public void Word_Lexer_State_BuiltIn_Tokens_As_Usual_Words()
{
string code = "Лев СтрДлина Прав";
var iterator = new SourceCodeIterator(code);
var state = new WordLexerState();
iterator.MoveToContent();
var lexem = state.ReadNextLexem(iterator);
Assert.True(lexem.Type == LexemType.Identifier);
Assert.True(lexem.Content == "Лев");
Assert.True(lexem.Token == Token.NotAToken);
iterator.MoveToContent();
lexem = state.ReadNextLexem(iterator);
Assert.True(lexem.Type == LexemType.Identifier);
Assert.True(lexem.Content == "СтрДлина");
Assert.True(lexem.Token == Token.NotAToken);
iterator.MoveToContent();
lexem = state.ReadNextLexem(iterator);
Assert.True(lexem.Type == LexemType.Identifier);
Assert.True(lexem.Content == "Прав");
Assert.True(lexem.Token == Token.NotAToken);
}
public override Lexem ReadNextLexem(SourceCodeIterator iterator)
{
if (iterator.CurrentSymbol == '<')
{
char next = iterator.PeekNext();
if (next == '>' || next == '=')
{
iterator.MoveNext();
iterator.MoveNext();
return(ExtractOperatorContent(iterator));
}
}
else if (iterator.CurrentSymbol == '>')
{
char next = iterator.PeekNext();
if (next == '=')
{
iterator.MoveNext();
iterator.MoveNext();
return(ExtractOperatorContent(iterator));
}
}
var lex = ExtractOperatorContent(iterator);
iterator.MoveNext();
return(lex);
}
public override Lexem ReadNextLexem(SourceCodeIterator iterator)
{
var lexem = base.ReadNextLexem(iterator);
lexem.Type = LexemType.Annotation;
return(lexem);
}
public void IteratorReturnsCorrectLine()
{
var code = "\r\nF = 1;\r\nD = 2;\r\nX = 3;\r\n";
var iterator = new SourceCodeIterator(code);
Assert.Equal(1, iterator.CurrentLine);
Assert.Equal(-1, iterator.CurrentColumn);
iterator.MoveToContent();
Assert.Equal(2, iterator.CurrentLine);
Assert.Equal(1, iterator.CurrentColumn);
Assert.Equal("F = 1;\r", iterator.GetCodeLine(2));
iterator.MoveNext();
Assert.Equal(2, iterator.CurrentColumn);
iterator.MoveNext();
Assert.Equal(3, iterator.CurrentColumn);
Assert.Equal('=', iterator.CurrentSymbol);
while (iterator.CurrentSymbol != 'D' && iterator.MoveNext())
{
;
}
Assert.Equal(1, iterator.CurrentColumn);
Assert.Equal(3, iterator.CurrentLine);
}
private Lexem CreateCommentLexem(SourceCodeIterator iterator)
{
return(new Lexem()
{
Content = iterator.GetContents().TrimEnd('\r', '\n'),
Type = LexemType.Comment,
Token = Token.NotAToken
});
}
public void Unclosed_String_Literal()
{
string code;
SourceCodeIterator iterator;
StringLexerState state = new StringLexerState();
code = " \"-just string ";
iterator = new SourceCodeIterator(code);
iterator.MoveToContent();
Assert.Throws <SyntaxErrorException>(() => state.ReadNextLexem(iterator));
}
public void Incorrect_NewLine_InString()
{
string code;
SourceCodeIterator iterator;
StringLexerState state = new StringLexerState();
code = @" ""-just
d|string """;
iterator = new SourceCodeIterator(code);
iterator.MoveToContent();
Assert.Throws <SyntaxErrorException>(() => state.ReadNextLexem(iterator));
}
public void Date_LexerState_Works_With_14_Numbers()
{
string code = " '12341212020202' ";
var iterator = new SourceCodeIterator(code);
iterator.MoveToContent();
var state = new DateLexerState();
var lex = state.ReadNextLexem(iterator);
Assert.True(lex.Type == LexemType.DateLiteral);
Assert.Equal("12341212020202", lex.Content);
}
public void Identifier_LexerState_Works_Fine()
{
string code = " \ndddddd-";
var iterator = new SourceCodeIterator(code);
var state = new WordLexerState();
iterator.MoveToContent();
var lexem = state.ReadNextLexem(iterator);
Assert.True(lexem.Type == LexemType.Identifier);
Assert.True(lexem.Content == "dddddd");
}
public override Lexem ReadNextLexem(SourceCodeIterator iterator)
{
iterator.MoveNext();
System.Diagnostics.Debug.Assert(iterator.CurrentSymbol == '/');
return(new Lexem()
{
Content = iterator.ReadToLineEnd(),
Type = LexemType.Comment,
Token = Token.NotAToken
});
}
public void NumberLiteral_State_Works_Fine()
{
string code = " 123.45 ";
var iterator = new SourceCodeIterator(code);
iterator.MoveToContent();
var state = new NumberLexerState();
var lex = state.ReadNextLexem(iterator);
Assert.True(lex.Type == LexemType.NumberLiteral);
Assert.Equal("123.45", lex.Content);
}
private static Lexem ExtractOperatorContent(SourceCodeIterator iterator)
{
Lexem lex;
var content = iterator.GetContents();
lex = new Lexem()
{
Type = LexemType.Operator,
Content = content,
Token = LanguageDef.GetToken(content)
};
return(lex);
}
public void SourceCode_Iterator_Basics()
{
string code = "Б = 1;";
var iterator = new SourceCodeIterator(code);
Assert.True(iterator.CurrentLine == 1);
Assert.True(iterator.CurrentSymbol == '\0');
Assert.True(iterator.PeekNext() == 'Б');
Assert.True(iterator.MoveNext());
Assert.True(iterator.CurrentLine == 1);
Assert.True(iterator.CurrentSymbol == 'Б');
Assert.True(iterator.PeekNext() == ' ');
Assert.True(iterator.Position == 0);
Assert.True(iterator.MoveNext());
Assert.True(iterator.CurrentLine == 1);
Assert.True(iterator.CurrentSymbol == ' ');
Assert.True(iterator.PeekNext() == '=');
Assert.True(iterator.Position == 1);
Assert.True(iterator.MoveNext());
Assert.True(iterator.CurrentLine == 1);
Assert.True(iterator.CurrentSymbol == '=');
Assert.True(iterator.PeekNext() == ' ');
Assert.True(iterator.Position == 2);
Assert.True(iterator.MoveNext());
Assert.True(iterator.CurrentLine == 1);
Assert.True(iterator.CurrentSymbol == ' ');
Assert.True(iterator.PeekNext() == '1');
Assert.True(iterator.Position == 3);
Assert.True(iterator.MoveNext());
Assert.True(iterator.CurrentLine == 1);
Assert.True(iterator.CurrentSymbol == '1');
Assert.True(iterator.PeekNext() == ';');
Assert.True(iterator.Position == 4);
Assert.True(iterator.MoveNext());
Assert.True(iterator.CurrentLine == 1);
Assert.True(iterator.CurrentSymbol == ';');
Assert.True(iterator.PeekNext() == '\0');
Assert.True(iterator.Position == 5);
Assert.False(iterator.MoveNext());
Assert.True(iterator.CurrentSymbol == '\0');
Assert.True(iterator.PeekNext() == '\0');
}
public void Wrong_Number_Literal()
{
string code = " 123.45.45 ";
var iterator = new SourceCodeIterator(code);
iterator.MoveToContent();
var state = new NumberLexerState();
Assert.Throws <SyntaxErrorException>(() => state.ReadNextLexem(iterator));
code = " 12jk";
iterator = new SourceCodeIterator(code);
iterator.MoveToContent();
Assert.Throws <SyntaxErrorException>(() => state.ReadNextLexem(iterator));
}
public override Lexem ReadNextLexem(SourceCodeIterator iterator)
{
iterator.MoveNext();
System.Diagnostics.Debug.Assert(iterator.CurrentSymbol == '/');
while (iterator.MoveNext())
{
if (iterator.CurrentSymbol == '\n')
{
return(CreateCommentLexem(iterator));
}
}
return(CreateCommentLexem(iterator));
}
public void Whitespace_Skipping()
{
string code = "Б \t\t=\n 1; ";
string check = "Б=1;";
var iterator = new SourceCodeIterator(code);
for (int i = 0; i < check.Length; i++)
{
iterator.MoveToContent();
Assert.True(check[i] == iterator.CurrentSymbol);
iterator.MoveNext();
}
Assert.True(iterator.CurrentLine == 2);
Assert.False(iterator.MoveNext());
}
public void Word_Literals_Processed_Correctly()
{
string code;
SourceCodeIterator iterator;
Lexem lex;
WordLexerState state = new WordLexerState();
code = " Истина Ложь Неопределено Null True False Undefined";
iterator = new SourceCodeIterator(code);
iterator.MoveToContent();
lex = state.ReadNextLexem(iterator);
Assert.Equal(LexemType.BooleanLiteral, lex.Type);
Assert.Equal("Истина", lex.Content);
iterator.MoveToContent();
lex = state.ReadNextLexem(iterator);
Assert.True(lex.Type == LexemType.BooleanLiteral);
Assert.Equal("Ложь", lex.Content);
iterator.MoveToContent();
lex = state.ReadNextLexem(iterator);
Assert.True(lex.Type == LexemType.UndefinedLiteral);
Assert.Equal("Неопределено", lex.Content);
iterator.MoveToContent();
lex = state.ReadNextLexem(iterator);
Assert.True(lex.Type == LexemType.NullLiteral);
Assert.Equal("Null", lex.Content);
iterator.MoveToContent();
lex = state.ReadNextLexem(iterator);
Assert.True(lex.Type == LexemType.BooleanLiteral);
Assert.Equal("True", lex.Content);
iterator.MoveToContent();
lex = state.ReadNextLexem(iterator);
Assert.True(lex.Type == LexemType.BooleanLiteral);
Assert.Equal("False", lex.Content);
iterator.MoveToContent();
lex = state.ReadNextLexem(iterator);
Assert.True(lex.Type == LexemType.UndefinedLiteral);
Assert.Equal("Undefined", lex.Content);
}
public override Lexem ReadNextLexem(SourceCodeIterator iterator)
{
iterator.MoveNext();
if (!iterator.MoveToContent())
{
throw CreateExceptionOnCurrentLine(MESSAGE_ANNOTATION_EXPECTED, iterator);
}
if (!char.IsLetter(iterator.CurrentSymbol))
{
throw CreateExceptionOnCurrentLine(MESSAGE_ANNOTATION_EXPECTED, iterator);
}
var lexem = _wordExtractor.ReadNextLexem(iterator);
lexem.Type = LexemType.Annotation;
return(lexem);
}
public void Preprocessor_Lexem_ProcessedCorrectly()
{
string code = "#Если #КонецЕсли";
var iterator = new SourceCodeIterator(code);
var wordParser = new PreprocessorDirectiveLexerState();
Lexem lex;
iterator.MoveToContent();
lex = wordParser.ReadNextLexem(iterator);
Assert.Equal(LexemType.PreprocessorDirective, lex.Type);
Assert.Equal("Если", lex.Content);
Assert.Equal(Token.If, lex.Token);
iterator.MoveToContent();
lex = wordParser.ReadNextLexem(iterator);
Assert.Equal(LexemType.PreprocessorDirective, lex.Type);
Assert.Equal("КонецЕсли", lex.Content);
Assert.Equal(Token.EndIf, lex.Token);
}
public void CurrentColumn_Calculation()
{
string code = @"А = 1;
Б = 2;
В = 7-11;
Г = 8";
var iterator = new SourceCodeIterator(code);
while (iterator.CurrentLine < 3)
{
if (!iterator.MoveNext())
{
throw new Xunit.Sdk.XunitException("Code is less than 3 lines");
}
}
Assert.True(iterator.CurrentLine == 3);
iterator.MoveToContent();
Assert.True(iterator.CurrentSymbol == 'В');
Assert.True(iterator.CurrentColumn == 13);
}
public void StringLiteral_LexerState_WorksFine()
{
string code;
SourceCodeIterator iterator;
Lexem lex;
StringLexerState state = new StringLexerState();
code = " \"-just string \"";
iterator = new SourceCodeIterator(code);
iterator.MoveToContent();
lex = state.ReadNextLexem(iterator);
Assert.True(lex.Type == LexemType.StringLiteral);
Assert.Equal("-just string ", lex.Content);
code = @" ""-just
|string """;
iterator = new SourceCodeIterator(code);
iterator.MoveToContent();
lex = state.ReadNextLexem(iterator);
Assert.True(lex.Type == LexemType.StringLiteral);
Assert.Equal("-just\nstring ", lex.Content);
code = @" ""-just "" ""string"" ""123""";
iterator = new SourceCodeIterator(code);
iterator.MoveToContent();
lex = state.ReadNextLexem(iterator);
Assert.True(lex.Type == LexemType.StringLiteral);
Assert.Equal("-just \nstring\n123", lex.Content);
code = @"""first line
|second line
// comment
|third line""";
iterator = new SourceCodeIterator(code);
iterator.MoveToContent();
lex = state.ReadNextLexem(iterator);
Assert.True(lex.Type == LexemType.StringLiteral);
Assert.Equal("first line\nsecond line\nthird line", lex.Content);
}
public void Retrieve_Line_Of_Code()
{
string code = @"А = 1;
Б = 2;
// comment
В = 7-11;
Г = 8";
var iterator = new SourceCodeIterator(code);
while (iterator.CurrentLine < 4)
{
if (!iterator.MoveNext())
{
throw new Xunit.Sdk.XunitException("Code is less than 4 lines");
}
}
Assert.True(iterator.CurrentLine == 4);
Assert.True(iterator.CurrentSymbol == '\n');
iterator.MoveNext();
Assert.True(iterator.GetCodeLine(4).Trim() == "В = 7-11;");
}
public override Lexem ReadNextLexem(SourceCodeIterator iterator)
{
bool isEndOfText = false;
char cs = '\0';
int currentLine = iterator.CurrentLine;
while (true)
{
if (!isEndOfText)
{
cs = iterator.CurrentSymbol;
}
if (SpecialChars.IsDelimiter(cs) || isEndOfText)
{
var content = iterator.GetContents();
Lexem lex;
if (LanguageDef.IsLogicalOperatorString(content))
{
lex = new Lexem()
{
Type = LexemType.Operator,
Token = LanguageDef.GetToken(content),
Content = content,
LineNumber = currentLine
};
}
else if (LanguageDef.IsBooleanLiteralString(content))
{
lex = new Lexem()
{
Type = LexemType.BooleanLiteral,
Content = content,
LineNumber = currentLine
};
}
else if (LanguageDef.IsUndefinedString(content))
{
lex = new Lexem()
{
Type = LexemType.UndefinedLiteral,
Content = content,
LineNumber = currentLine
};
}
else if (LanguageDef.IsNullString(content))
{
lex = new Lexem()
{
Type = LexemType.NullLiteral,
Content = content,
LineNumber = currentLine
};
}
else
{
lex = new Lexem()
{
Type = LexemType.Identifier,
Content = content,
Token = LanguageDef.GetToken(content),
LineNumber = currentLine
};
if (LanguageDef.IsBuiltInFunction(lex.Token))
{
iterator.SkipSpaces();
if (iterator.CurrentSymbol != '(')
{
lex.Token = Token.NotAToken;
}
}
}
return(lex);
}
if (!iterator.MoveNext())
{
isEndOfText = true;
}
}
}
public SyntaxErrorException CreateExceptionOnCurrentLine(string message, SourceCodeIterator iterator)
{
var cp = iterator.GetPositionInfo();
return(new SyntaxErrorException(cp, message));
}
public override Lexem ReadNextLexem(SourceCodeIterator iterator)
{
return(Lexem.Empty());
}
public void Operators_Lexer_State()
{
string code = " + - * / < > <= >= <> % ,.()[]";
var iterator = new SourceCodeIterator(code);
var state = new OperatorLexerState();
Lexem lex;
iterator.MoveToContent();
lex = state.ReadNextLexem(iterator);
Assert.True(lex.Type == LexemType.Operator);
Assert.Equal("+", lex.Content);
iterator.MoveToContent();
lex = state.ReadNextLexem(iterator);
Assert.True(lex.Type == LexemType.Operator);
Assert.Equal("-", lex.Content);
iterator.MoveToContent();
lex = state.ReadNextLexem(iterator);
Assert.True(lex.Type == LexemType.Operator);
Assert.Equal("*", lex.Content);
iterator.MoveToContent();
lex = state.ReadNextLexem(iterator);
Assert.True(lex.Type == LexemType.Operator);
Assert.Equal("/", lex.Content);
iterator.MoveToContent();
lex = state.ReadNextLexem(iterator);
Assert.True(lex.Type == LexemType.Operator);
Assert.Equal("<", lex.Content);
iterator.MoveToContent();
lex = state.ReadNextLexem(iterator);
Assert.True(lex.Type == LexemType.Operator);
Assert.Equal(">", lex.Content);
iterator.MoveToContent();
lex = state.ReadNextLexem(iterator);
Assert.True(lex.Type == LexemType.Operator);
Assert.Equal("<=", lex.Content);
iterator.MoveToContent();
lex = state.ReadNextLexem(iterator);
Assert.True(lex.Type == LexemType.Operator);
Assert.Equal(">=", lex.Content);
iterator.MoveToContent();
lex = state.ReadNextLexem(iterator);
Assert.True(lex.Type == LexemType.Operator);
Assert.Equal("<>", lex.Content);
iterator.MoveToContent();
lex = state.ReadNextLexem(iterator);
Assert.True(lex.Type == LexemType.Operator);
Assert.Equal("%", lex.Content);
iterator.MoveToContent();
lex = state.ReadNextLexem(iterator);
Assert.True(lex.Type == LexemType.Operator);
Assert.Equal(",", lex.Content);
iterator.MoveToContent();
lex = state.ReadNextLexem(iterator);
Assert.True(lex.Type == LexemType.Operator);
Assert.Equal(".", lex.Content);
iterator.MoveToContent();
lex = state.ReadNextLexem(iterator);
Assert.True(lex.Type == LexemType.Operator);
Assert.Equal("(", lex.Content);
iterator.MoveToContent();
lex = state.ReadNextLexem(iterator);
Assert.True(lex.Type == LexemType.Operator);
Assert.Equal(")", lex.Content);
iterator.MoveToContent();
lex = state.ReadNextLexem(iterator);
Assert.True(lex.Type == LexemType.Operator);
Assert.Equal("[", lex.Content);
iterator.MoveToContent();
lex = state.ReadNextLexem(iterator);
Assert.True(lex.Type == LexemType.Operator);
Assert.Equal("]", lex.Content);
}
abstract public Lexem ReadNextLexem(SourceCodeIterator iterator);
