public static void TestManyString()
{
var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <ManyString> >());
Assert.False(lexerRes.IsError);
var lexer = lexerRes.Result;
var string1 = "\"hello \\\"world \"";
var expectString1 = "\"hello \"world \"";
var string2 = "'that''s it'";
var expectString2 = "'that's it'";
var source1 = $"{string1} {string2}";
var r = lexer.Tokenize(source1);
Assert.True(r.IsOk);
Assert.Equal(3, r.Tokens.Count);
var tok1 = r.Tokens[0];
Assert.Equal(ManyString.STRING, tok1.TokenID);
Assert.Equal(expectString1, tok1.Value);
Assert.Equal('"', tok1.StringDelimiter);
var tok2 = r.Tokens[1];
Assert.Equal(ManyString.STRING, tok2.TokenID);
Assert.Equal(expectString2, tok2.Value);
Assert.Equal('\'', tok2.StringDelimiter);
}
public static void Issue218()
{
var lexerResult = LexerBuilder.BuildLexer <Token218>();
Assert.True(lexerResult.IsOk);
var lexer = lexerResult.Result;
var result = lexer.Tokenize("a = 0.0;");
Assert.True(result.IsOk);
var tokens = result.Tokens;
var dump = string.Join(" ", tokens.Select(x => x.ToString()));
int n = 0;
int count = 0;
while ((n = dump.IndexOf("<<EOS>>", n)) != -1)
{
n++;
count++;
}
Console.WriteLine(dump);
Assert.Equal(1, count);
var eoss = tokens.Where(x => x.IsEOS);
Assert.Single(eoss);
}
private static void Main(string[] args)
{
if (args.Length < 1 || args.Any(arg => !File.Exists(arg)))
{
Console.WriteLine(
$"Usage {Path.GetFileNameWithoutExtension(Assembly.GetExecutingAssembly().Location)} codefile.llama [, codefile2.llama, ...]"
);
}
var time = Stopwatch.StartNew();
var sourceFilePath = args[0];
var source = string.Join("\n", args.Select(File.ReadAllText));
var lexer = LexerBuilder.BuildLlamaLexer();
var parser = new ParseContext(LlamaParseStore.Instance, lexer, source);
var document = parser.ReadNode <LlamaDocument>();
Console.WriteLine($"Parsing time: {time.Elapsed.TotalMilliseconds:F2} ms");
time.Restart();
var functionDeclarations = document.Functions.Select(fun => fun.Declaration).ToArray();
var compiler = new Compiler(new LlamaCompilerStore(), new LinkerFactory(), document.Imports, functionDeclarations);
foreach (var functionImplementation in document.Functions)
{
compiler.AddFunction(functionImplementation);
}
var codeBlob = compiler.Code;
var linker = (Linker)compiler.LinkingInfo;
Console.WriteLine($"Compilation time: {time.Elapsed.TotalMilliseconds:F2} ms");
time.Restart();
var pe32PlusExeBuilder = new ExecutableBuilder
{
Subsystem = Subsystem.WindowsCui // Change to WindowsGui to make a Window application (without console)
};
linker.LinkPreBuild(pe32PlusExeBuilder);
var mainOffset = linker.GetCodeOffsetOfKnownFunction("Main");
if (mainOffset < 0)
{
Console.WriteLine("No main function");
return;
}
pe32PlusExeBuilder.AddAdditionalSection("A\nB\t\0", 200, SectionCharacteristics.MemWrite | SectionCharacteristics.MemRead);
var peResult = pe32PlusExeBuilder.Build((uint)codeBlob.Length, (uint)mainOffset);
codeBlob.CopyTo(peResult.GetCodeSectionBuffer());
linker.LinkPostBuild(peResult);
Console.WriteLine($"Linking time: {time.Elapsed.TotalMilliseconds:F2} ms");
time.Restart();
using var fileStream = File.Open(Path.ChangeExtension(sourceFilePath, "exe"), FileMode.Create);
peResult.Finish(fileStream);
}
/// <summary>
/// Handles construction of the elements yielded by the
/// <see cref="ParserBuilder"/> with the <paramref name="input"/>
/// provided.
/// </summary>
/// <param name="input">The <see cref="Tuple{T1, T2, T3}"/> which provides the
/// <see cref="ParserCompiler"/>, <see cref="TokenSymbolBuilder"/> and
/// <see cref="IIntermediateAssembly"/> necessary to perform the operation.</param>
/// <returns>A <see cref="Tuple{T1, T2, T3}"/> which yields the
/// <see cref="IIntermediateInterfaceType"/>, <see cref="IIntermediateClassType"/>
/// for the result parser of the language, and
/// <see cref="LexerBuilder"/> of the language which denotes the class
/// and interface of the tokenizer.</returns>
public Tuple <IIntermediateInterfaceType, IIntermediateClassType, LexerBuilder> Build(Tuple <ParserCompiler, TokenSymbolBuilder, IIntermediateAssembly, RegularLanguageDFAState> input)
{
this.compiler = input.Item1;
this._tokenSymbolBuilder = input.Item2;
this._assembly = input.Item3;
this._identityManager = (IIntermediateCliManager)this._assembly.IdentityManager;
this._parserInterface = _assembly.DefaultNamespace.Parts.Add().Interfaces.Add("I{0}", compiler.Source.Options.ParserName);
this._parserClass = _assembly.DefaultNamespace.Parts.Add().Classes.Add(compiler.Source.Options.ParserName);
this._parserClass.Assembly.ScopeCoercions.Add("System.Linq");
this._genericSymbolStreamBuilder = new GenericSymbolStreamBuilder();
this._symbolStreamBuilder = new SymbolStreamBuilder();
this._lexerBuilder = new LexerBuilder();
this._parserClass.AccessLevel = AccessLevelModifiers.Internal;
this._parserInterface.AccessLevel = AccessLevelModifiers.Public;
this._lexerCore = input.Item4;
this._parserState = this._parserClass.Fields.Add(new TypedName("_state", _assembly.IdentityManager.ObtainTypeReference(RuntimeCoreType.Int32)));
this._parserState.AccessLevel = AccessLevelModifiers.Private;
this._parserStateProp = this._parserInterface.Properties.Add(new TypedName("State", this._parserState.FieldType), true, false);
this._parserStatePropImpl = this._parserClass.Properties.Add(new TypedName("State", this._parserState.FieldType), true, true);
this._parserStatePropImpl.AccessLevel = AccessLevelModifiers.Public;
this._parserStatePropImpl.SetMethod.AccessLevel = AccessLevelModifiers.Private;
this._parserStatePropImpl.GetMethod.Return(this._parserState.GetReference());
this._parserStatePropImpl.SetMethod.Assign(this._parserState.GetReference(), this._parserStatePropImpl.SetMethod.ValueParameter.GetReference());
this._parserClass.ImplementedInterfaces.ImplementInterfaceQuick(this._parserInterface);
return(Tuple.Create(this.ParserInterface, this.ParserClass, this._lexerBuilder));
}
public override bool TryMatchCurrent(LexerBuilder lexer)
{
var prev = lexer.Previous;
var first = lexer.Current;
// check for keywords
if (lexer.IsDelimiter(prev, true) && char.IsLetter(first))
{
// can be a keyword...
var sb = new StringBuilder();
sb.Append(lexer.Current);
while (!lexer.EndOfInput && char.IsLetter(lexer.PeekNext()))
{
sb.Append(lexer.Current);
}
if (keywords.Contains(sb.ToString()))
{
if (!lexer.EndOfInput)
{
lexer.RollbackBy(1);
}
lexer.Commit();
return(true);
}
return(false);
}
else
{
return(false);
}
}
public void TestExtensions()
{
var lexerRes =
LexerBuilder.BuildLexer(new BuildResult <ILexer <Extensions> >(), ExtendedGenericLexer.AddExtension);
Assert.False(lexerRes.IsError);
var lexer = lexerRes.Result as GenericLexer <Extensions>;
Assert.NotNull(lexer);
var tokens = lexer.Tokenize("20.02.2018 3.14").ToList();
Assert.Equal(3, tokens.Count);
Assert.Equal(Extensions.DATE, tokens[0].TokenID);
Assert.Equal("20.02.2018", tokens[0].Value);
Assert.Equal(Extensions.DOUBLE, tokens[1].TokenID);
Assert.Equal("3.14", tokens[1].Value);
tokens = lexer.Tokenize("'that''s it'").ToList();
Assert.Equal(2, tokens.Count);
var tok = tokens[0];
Assert.Equal(Extensions.CHAINE, tok.TokenID);
Assert.Equal("'that's it'", tokens[0].Value);
tokens = lexer.Tokenize("'et voilà'").ToList();
Assert.Equal(2, tokens.Count);
tok = tokens[0];
Assert.Equal(Extensions.CHAINE, tok.TokenID);
Assert.Equal("'et voilà'", tokens[0].Value);
}
public override bool TryMatchCurrent(LexerBuilder lexer)
{
if (lexer.Current == '/')
{
lexer.PeekNext();
if (lexer.Current == '/')
{
// line comment. read to end of line or file.
do
{
lexer.Commit();
lexer.PeekNext();
}while (!lexer.EndOrNewLine);
return(true);
}
else if (lexer.Current == '*')
{
// block comment, read until end of file or closing '*/'
lexer.PeekNext();
do
{
lexer.PeekNext();
lexer.Commit();
}while (!lexer.EndOfInput && !(lexer.Current == '/' && lexer.Previous == '*'));
return(true);
}
}
return(false);
}
public void Test__Numeric()
{
// Arrange
var lexer = LexerBuilder.New <BaseNode>()
.Init()
.WithAmbiguityResolverEnum(AmbiguityResolverEnum.FirstMatch)
.NilNode(() => new NilNode())
.WithHandler(new Regex("[\\d]+"), opt => opt.Yield(opt.Buffer, new NumericNode {
Value = decimal.Parse(opt.Buffer)
}))
.WithHandler(new Regex("[+-\\/*]"), opt => opt.Yield(opt.Buffer, new OpNode {
Operator = opt.Buffer[0]
}))
.WithHandler(new Regex("[\\s]+"), opt => opt.Dispose(opt.Buffer))
.WithHandler(new Regex("[\\w]+"), opt => opt.SetBuffer(string.Empty))
.WithHandler(new Regex("$"), opt => opt.Stop())
.FinalizeHandlers()
.Build();
// Act
var nodes = lexer.Process(new MemoryStream(Encoding.Default.GetBytes("12 * 23 + 34 / 45 - 56 Error 12 * 23 + 34 / 45 - 56")));
// Assert
Assert.Equal(18, nodes.Count());
}
public void TestCharTokenDelimiterConflict()
{
var res = LexerBuilder.BuildLexer(new BuildResult <ILexer <CharTokensConflicts> >());
Assert.True(res.IsError);
Assert.Equal(2, res.Errors.Count);
}
public void TestIssue114()
{
var res = LexerBuilder.BuildLexer(new BuildResult <ILexer <Issue114> >());
Assert.False(res.IsError);
var lexer = res.Result as GenericLexer <Issue114>;
var r = lexer?.Tokenize("// /&");
Assert.True(r.IsError);
Assert.Equal('&', r.Error.UnexpectedChar);
r = lexer?.Tokenize("/&");
Assert.Equal('&', r.Error.UnexpectedChar);
r = lexer?.Tokenize("&/");
Assert.True(r.IsError);
Assert.Equal('&', r.Error.UnexpectedChar);
r = lexer?.Tokenize("// &");
Assert.True(r.IsError);
Assert.Equal('&', r.Error.UnexpectedChar);
}
private static void TestChars()
{
var res = LexerBuilder.BuildLexer(new BuildResult <ILexer <CharTokens> >());
if (res.IsOk)
{
var lexer = res.Result as GenericLexer <CharTokens>;
var dump = lexer.ToString();
var graph = lexer.ToGraphViz();
Console.WriteLine(graph);
var source = "'\\''";
Console.WriteLine(source);
var res2 = lexer.Tokenize(source);
Console.WriteLine($"{res2.IsOk} - {res2.Tokens[0].Value}");
var sourceU = "'\\u0066'";
var res3 = lexer.Tokenize(sourceU);
Console.WriteLine($"{res3.IsOk} - {res3.Tokens[0].Value}");
}
else
{
var errors = string.Join('\n', res.Errors.Select(e => e.Level + " - " + e.Message).ToList());
Console.WriteLine("error building lexer : ");
Console.WriteLine(errors);
}
}
public void TestCustomIdWithOther()
{
var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <CustomId> >());
Assert.False(lexerRes.IsError);
var lexer = lexerRes.Result;
var r = lexer.Tokenize("_b_ -C-");
Assert.True(r.IsOk);
Assert.Equal(5, r.Tokens.Count);
var tok1 = r.Tokens[0];
Assert.Equal(CustomId.OTHER, tok1.TokenID);
Assert.Equal("_", tok1.Value);
var tok2 = r.Tokens[1];
Assert.Equal(CustomId.ID, tok2.TokenID);
Assert.Equal("b_", tok2.Value);
var tok3 = r.Tokens[2];
Assert.Equal(CustomId.OTHER, tok3.TokenID);
Assert.Equal("-", tok3.Value);
var tok4 = r.Tokens[3];
Assert.Equal(CustomId.ID, tok4.TokenID);
Assert.Equal("C-", tok4.Value);
}
public void TestIssue210()
{
var lexResult = LexerBuilder.BuildLexer <Issue210Token>(Issue210Extensions.AddExtensions);
Assert.True(lexResult.IsOk);
var lexer = lexResult.Result;
Assert.NotNull(lexer);
var r = lexer.Tokenize("?");
Assert.True(r.IsOk);
var l = ToTokens2(r);
Assert.Equal("QMARK[?]EOF[]", l);
r = lexer.Tokenize("?special?");
Assert.True(r.IsOk);
l = ToTokens2(r);
Assert.Equal("SPECIAL[special]EOF[]", l);
r = lexer.Tokenize("x?x");
Assert.True(r.IsOk);
l = ToTokens2(r);
Assert.Equal("X[x]QMARK[?]X[x]EOF[]", l);
r = lexer.Tokenize("??");
Assert.True(r.IsOk);
l = ToTokens2(r);
Assert.Equal("SPECIAL[]EOF[]", l);
}
public void TestIssue114()
{
var res = LexerBuilder.BuildLexer(new BuildResult <ILexer <Issue114> >());
Assert.False(res.IsError);
var lexer = res.Result as GenericLexer <Issue114>;
var error = Assert.Throws <LexerException>(() =>
{
lexer?.Tokenize("// /&").ToList();
});
Assert.Equal('&', error.Error.UnexpectedChar);
error = Assert.Throws <LexerException>(() =>
{
lexer?.Tokenize("/&").ToList();
});
Assert.Equal('&', error.Error.UnexpectedChar);
error = Assert.Throws <LexerException>(() =>
{
lexer?.Tokenize("&/").ToList();
});
Assert.Equal('&', error.Error.UnexpectedChar);
error = Assert.Throws <LexerException>(() =>
{
lexer?.Tokenize("// &").ToList();
});
Assert.Equal('&', error.Error.UnexpectedChar);
}
public void TestSelfEscapedString()
{
var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <SelfEscapedString> >());
Assert.False(lexerRes.IsError);
var lexer = lexerRes.Result as GenericLexer <SelfEscapedString>;
Assert.NotNull(lexer);
var r = lexer.Tokenize("'that''s it'");
Assert.True(r.IsOk);
var tokens = r.Tokens;
Assert.Equal(2, tokens.Count);
var tok = tokens[0];
Assert.Equal(SelfEscapedString.STRING, tok.TokenID);
Assert.Equal("'that's it'", tokens[0].Value);
r = lexer.Tokenize("'et voilà'");
Assert.True(r.IsOk);
tokens = r.Tokens;
Assert.Equal(2, tokens.Count);
tok = tokens[0];
Assert.Equal(SelfEscapedString.STRING, tok.TokenID);
Assert.Equal("'et voilà'", tokens[0].Value);
}
public void TestSameIntValuesError()
{
var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <SameIntValuesError> >());
Assert.True(lexerRes.IsError);
Assert.Null(lexerRes.Result);
Assert.Single(lexerRes.Errors);
Assert.Equal(ErrorCodes.LEXER_SAME_VALUE_USED_MANY_TIME, lexerRes.Errors.First().Code);
}
public void TestInnerMultiComment()
{
var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <CommentsToken> >());
Assert.False(lexerRes.IsError);
var lexer = lexerRes.Result as GenericLexer <CommentsToken>;
var dump = lexer.ToString();
var code = @"1
2 /* inner */ 3
4
";
var r = lexer.Tokenize(code);
Assert.True(r.IsOk);
var tokens = r.Tokens;
Assert.Equal(6, tokens.Count);
var token1 = tokens[0];
var token2 = tokens[1];
var token3 = tokens[2];
var token4 = tokens[3];
var token5 = tokens[4];
Assert.Equal(CommentsToken.INT, token1.TokenID);
Assert.Equal("1", token1.Value);
Assert.Equal(0, token1.Position.Line);
Assert.Equal(0, token1.Position.Column);
Assert.Equal(CommentsToken.INT, token2.TokenID);
Assert.Equal("2", token2.Value);
Assert.Equal(1, token2.Position.Line);
Assert.Equal(0, token2.Position.Column);
Assert.Equal(CommentsToken.COMMENT, token3.TokenID);
Assert.Equal(@" inner ", token3.Value);
Assert.Equal(1, token3.Position.Line);
Assert.Equal(2, token3.Position.Column);
Assert.Equal(CommentsToken.INT, token4.TokenID);
Assert.Equal("3", token4.Value);
Assert.Equal(1, token4.Position.Line);
Assert.Equal(14, token4.Position.Column);
Assert.Equal(CommentsToken.INT, token5.TokenID);
Assert.Equal("4", token5.Value);
Assert.Equal(2, token5.Position.Line);
Assert.Equal(0, token5.Position.Column);
}
public void TestBadEmptyStringDelimiter()
{
var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <BadEmptyStringDelimiter> >());
Assert.True(lexerRes.IsError);
Assert.Single(lexerRes.Errors);
var error = lexerRes.Errors[0];
Assert.Equal(ErrorLevel.FATAL, error.Level);
Assert.Equal(ErrorCodes.LEXER_STRING_DELIMITER_MUST_BE_1_CHAR, error.Code);
}
public void TestBadEscapeStringDelimiterTooLong()
{
var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <BadEscapeStringDelimiterTooLong> >());
Assert.True(lexerRes.IsError);
Assert.Single(lexerRes.Errors);
var error = lexerRes.Errors[0];
Assert.Equal(ErrorLevel.FATAL, error.Level);
Assert.Contains("must be 1 character length", error.Message);
}
public void Setup()
{
content = File.ReadAllText("test.json");
var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <JsonTokenGeneric> >());
if (lexerRes != null)
{
BenchedLexer = lexerRes.Result;
}
}
public void TestBadEscapeStringDelimiterLetter()
{
var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <BadEscapeStringDelimiterLetter> >());
Assert.True(lexerRes.IsError);
Assert.Single(lexerRes.Errors);
var error = lexerRes.Errors[0];
Assert.Equal(ErrorLevel.FATAL, error.Level);
Assert.Contains("can not start with a letter", error.Message);
}
public void TestBadLetterStringDelimiter()
{
var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <BadLetterStringDelimiter> >());
Assert.True(lexerRes.IsError);
Assert.Single(lexerRes.Errors);
var error = lexerRes.Errors[0];
Assert.Equal(ErrorLevel.FATAL, error.Level);
Assert.Equal(ErrorCodes.LEXER_STRING_DELIMITER_CANNOT_BE_LETTER_OR_DIGIT, error.Code);
}
private static void TestDoubleExponent()
{
var lex = LexerBuilder.BuildLexer <DoubleExponent>(AddExponentExtension);
if (lex.IsOk)
{
var one = lex.Result.Tokenize("2.0E+2");
;
var two = lex.Result.Tokenize("4.0e-2");
;
}
}
protected GeneratorArgumentParser(string input)
{
_input = input;
var buildResult = LexerBuilder.BuildLexer(new BuildResult <ILexer <GeneratorTokenType> >());
if (buildResult.IsError)
{
throw new ApplicationException("Lexer failed to build");
}
_lexer = buildResult.Result;
}
private static void benchLexer()
{
var content = File.ReadAllText("test.json");
var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <JsonTokenGeneric> >());
ILexer <JsonTokenGeneric> BenchedLexer = null;
if (lexerRes != null)
{
BenchedLexer = lexerRes.Result;
BenchedLexer.Tokenize(content);
}
}
public CanParseGenerics()
{
var lexer = LexerBuilder.Build()
.Match("nr", @"\d+", match => new Number {
Value = int.Parse(match.Value)
})
.Match("txt", @".+", match => new Text {
Value = match.Value
});
_subject = ParserBuilder.Build(lexer)
.Match <Number, Text>((nr, t) => new KeyValuePair <Number, Text>(nr, t), MatchOptions.AtLeastOne)
.Create <KeyValuePair <Number, Text> >();
}
public void MixedErrors()
{
var lexerRes10 = LexerBuilder.BuildLexer(new BuildResult <ILexer <CommentsTokenError10> >());
Assert.True(lexerRes10.IsError);
Assert.Equal(4, lexerRes10.Errors.Count);
var expectedErrors = new[]
{
"too many comment lexem", "too many multi-line comment lexem", "too many single-line comment lexem",
"comment lexem can't be used together with single-line or multi-line comment lexems"
};
foreach (var expectedError in expectedErrors)
{
Assert.True(lexerRes10.Errors.Exists(x => x.Level == ErrorLevel.FATAL && x.Message == expectedError));
}
expectedErrors = new[] { "too many multi-line comment lexem", "too many single-line comment lexem" };
var lexerRes9 = LexerBuilder.BuildLexer(new BuildResult <ILexer <CommentsTokenError9> >());
Assert.True(lexerRes9.IsError);
Assert.Equal(2, lexerRes9.Errors.Count);
foreach (var expectedError in expectedErrors)
{
Assert.True(lexerRes9.Errors.Exists(x => x.Level == ErrorLevel.FATAL && x.Message == expectedError));
}
expectedErrors = new[] { "too many multi-line comment lexem", "comment lexem can't be used together with single-line or multi-line comment lexems" };
var lexerRes8 = LexerBuilder.BuildLexer(new BuildResult <ILexer <CommentsTokenError8> >());
Assert.True(lexerRes8.IsError);
Assert.Equal(2, lexerRes8.Errors.Count);
foreach (var expectedError in expectedErrors)
{
Assert.True(lexerRes8.Errors.Exists(x => x.Level == ErrorLevel.FATAL && x.Message == expectedError));
}
expectedErrors = new[] { "too many single-line comment lexem", "comment lexem can't be used together with single-line or multi-line comment lexems" };
var lexerRes7 = LexerBuilder.BuildLexer(new BuildResult <ILexer <CommentsTokenError7> >());
Assert.True(lexerRes7.IsError);
Assert.Equal(2, lexerRes7.Errors.Count);
foreach (var expectedError in expectedErrors)
{
Assert.True(lexerRes7.Errors.Exists(x => x.Level == ErrorLevel.FATAL && x.Message == expectedError));
}
}
public void TestAlphaNumId()
{
var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <AlphaNumId> >());
Assert.False(lexerRes.IsError);
var lexer = lexerRes.Result;
var r = lexer.Tokenize("alpha123");
Assert.True(r.IsOk);
Assert.Equal(2, r.Tokens.Count);
var tok = r.Tokens[0];
Assert.Equal(AlphaNumId.ID, tok.TokenID);
Assert.Equal("alpha123", tok.StringWithoutQuotes);
}
public void TestIgnoreEOL()
{
var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <IgnoreEOL> >());
Assert.False(lexerRes.IsError);
var lexer = lexerRes.Result;
var r = lexer.Tokenize(" \n ");
Assert.True(r.IsOk);
Assert.Equal(2, r.Tokens.Count);
var tok = r.Tokens[0];
Assert.Equal(IgnoreEOL.EOL, tok.TokenID);
Assert.Equal("\n", tok.Value);
}
public void TestWhiteSpace()
{
var lexerRes = LexerBuilder.BuildLexer(new BuildResult <ILexer <WhiteSpace> >());
Assert.False(lexerRes.IsError);
var lexer = lexerRes.Result;
var r = lexer.Tokenize(" \t ");
Assert.True(r.IsOk);
Assert.Equal(2, r.Tokens.Count);
var tok = r.Tokens[0];
Assert.Equal(WhiteSpace.TAB, tok.TokenID);
Assert.Equal("\t", tok.Value);
}
