public async Task TestTokenIteration(IList<Token> expectedTokens)
{
using (var tokenIterator = new AsyncLATokenIterator(expectedTokens.AsAsync()))
{
for (var i = 0; i < expectedTokens.Count; i++)
{
Assert.That(await tokenIterator.MoveNextAsync(), Is.True, "Move next i={0}", i);
var current = await tokenIterator.CurrentAsync();
Assert.That(current, Is.EqualTo(expectedTokens[i]), "Current i={0} expected={1} is={2}",
i, expectedTokens[i], current);
var la = await tokenIterator.LookAheadAsync();
var expectedLA = i + 1 < expectedTokens.Count ? expectedTokens[i + 1] : Token.EOF;
Assert.That(la, Is.EqualTo(expectedLA), "Lookahead i={0} expected={1} is={2}",
i, expectedLA, la);
}
}
}
public async Task TestSupportResetting(IList <Item> expectedTokens)
{
using (var tokenIterator = new AsyncLATokenIterator(expectedTokens.AsAsync()))
{
if (tokenIterator.SupportsResetting)
{
var isEmpty = true;
while (await tokenIterator.MoveNextAsync())
{
isEmpty = false;
}
tokenIterator.Reset();
Assert.That(tokenIterator.MoveNextAsync(), Is.EqualTo(isEmpty), "Has element after resetting is not empty");
}
else
{
Assert.That(tokenIterator.Reset, Throws.InvalidOperationException);
}
}
}
public async Task TestTokenIteration(IList <Item> expectedTokens)
{
using (var tokenIterator = new AsyncLATokenIterator(expectedTokens.AsAsync()))
{
for (var i = 0; i < expectedTokens.Count; i++)
{
Assert.That(await tokenIterator.MoveNextAsync(), Is.True, "Move next i={0}", i);
var current = await tokenIterator.CurrentAsync();
Assert.That(current, Is.EqualTo(expectedTokens[i]), "Current i={0} expected={1} is={2}",
i, expectedTokens[i], current);
var la = await tokenIterator.LookAheadAsync();
var expectedLA = i + 1 < expectedTokens.Count ? expectedTokens[i + 1] : Item.EOF;
Assert.That(la, Is.EqualTo(expectedLA), "Lookahead i={0} expected={1} is={2}",
i, expectedLA, la);
}
}
}
public async Task TestSupportResetting(IList<Token> expectedTokens)
{
using (var tokenIterator = new AsyncLATokenIterator(expectedTokens.AsAsync()))
{
if (tokenIterator.SupportsResetting)
{
var isEmpty = true;
while (await tokenIterator.MoveNextAsync())
{
isEmpty = false;
}
tokenIterator.Reset();
Assert.That(tokenIterator.MoveNextAsync(), Is.EqualTo(isEmpty), "Has element after resetting is not empty");
}
else
{
Assert.That(tokenIterator.Reset, Throws.InvalidOperationException);
}
}
}
private static async Task MainAsync(string[] args, CancellationToken token)
{
//
// the following program produces a parse table for the following grammar
// for infix expressions, and appropriately applies operator precedence of
// the + - * / operators, otherwise evaluating the leftmost operations first
//
// S' -> e
// e -> i
// e -> ( e )
// e -> e * e
// e -> e / e
// e -> e + e
// e -> e - e
//
var grammar = new Grammar(
new[] {"S'", "e", "+", "-", "*", "/", "i", "(", ")"},
new PrecedenceGroup(Derivation.None,
//S' -> e
new Production(0, (_, x) => x[0], 1),
//e -> i
new Production(1, (_, x) => x.Length == 1 ? x[0] : null, 6),
//e -> ( e )
new Production(1,
(_, x) => x[1].ContentType == ContentType.Nested ? x[1].Nested : null,
7, 1, 8)
),
new PrecedenceGroup(Derivation.LeftMost,
//e -> e * e
new Production(1, RewriteConstBinaryExpr, 1, 4, 1),
//e -> e / e
new Production(1, RewriteConstBinaryExpr, 1, 5, 1)
),
// productions are left associative and bind less tightly than * or /
new PrecedenceGroup(Derivation.LeftMost,
//e -> e + e
new Production(1, RewriteConstBinaryExpr, 1, 2, 1),
//e -> e - e
new Production(1, RewriteConstBinaryExpr, 1, 3, 1)
)
);
// generate the parse table
var parser = new Parser(grammar);
var debugger = new Debug(parser, Console.Write, Console.Error.Write);
debugger.DumpParseTable();
debugger.Flush();
var parseTime = System.Diagnostics.Stopwatch.StartNew();
#if DEBUG
// (24 / 12) + 2 * (3-4)
var inputSource = new[]
{
new Token(6, 2),
new Token(2, "+"),
new Token(7, "("),
new Token(6, 0),
new Token(4, "*"),
new Token(6, int.MaxValue),
new Token(8, ")")
};
#else
var inputSource = TestLarge();
#endif
Token result;
using (var tokenIterator = new AsyncLATokenIterator(inputSource.AsAsync()))
{
result = await parser.ParseInputAsync(tokenIterator, debugger, allowRewriting: true);
}
parseTime.Stop();
var timeElapsed = string.Format("{0} ms", parseTime.Elapsed.TotalMilliseconds);
debugger.WriteFinalToken(
string.Format("Accept ({0}): ", timeElapsed),
string.Format("Error while parsing ({0}): ", timeElapsed),
result);
}
private static async Task MainAsync(string[] args, CancellationToken cancellationToken)
{
//
// the following program produces a parse table for the following grammar
// for infix expressions, and appropriately applies operator precedence of
// the + - * / operators, otherwise evaluating the leftmost operations first
//
// S' -> e
// e -> i
// e -> ( e )
// e -> e * e
// e -> e / e
// e -> e + e
// e -> e - e
//
var grammar = new Grammar(
Symbols,
new PrecedenceGroup(Derivation.None,
//S' -> e
new Production((int)SymbolId.Result, (_, x) => x[0], (int)SymbolId.Expr),
//e -> i
new Production((int)SymbolId.Expr, (_, x) => x.Length == 1 ? x[0] : null, (int)SymbolId.Integer),
//e -> ( e )
new Production((int)SymbolId.Expr,
(_, x) => x[1].ContentType == ContentType.Nested ? x[1].Nested : null,
(int)SymbolId.LeftParen, (int)SymbolId.Expr, (int)SymbolId.RightParen)
),
new PrecedenceGroup(Derivation.LeftMost,
//e -> e * e
new Production((int)SymbolId.Expr, RewriteConstBinaryExpr, (int)SymbolId.Expr, (int)SymbolId.Times, (int)SymbolId.Expr),
//e -> e / e
new Production((int)SymbolId.Expr, RewriteConstBinaryExpr, (int)SymbolId.Expr, (int)SymbolId.Divide, (int)SymbolId.Expr)
),
// productions are left associative and bind less tightly than * or /
new PrecedenceGroup(Derivation.LeftMost,
//e -> e + e
new Production((int)SymbolId.Expr, RewriteConstBinaryExpr, (int)SymbolId.Expr, (int)SymbolId.Plus, (int)SymbolId.Expr),
//e -> e - e
new Production((int)SymbolId.Expr, RewriteConstBinaryExpr, (int)SymbolId.Expr, (int)SymbolId.Minus, (int)SymbolId.Expr)
)
);
// generate the parse table
var parser = new Parser(grammar);
var debugger = new Debug(parser, Console.Write, Console.Error.Write);
debugger.DumpParseTable();
debugger.Flush();
var parseTime = System.Diagnostics.Stopwatch.StartNew();
Collection <PSParseError> syntaxErrors;
var inputSource = PSParser.Tokenize("(24 / 12) + 2 * (3-4)", out syntaxErrors).Select(PSTokenToItem);
Item result;
using (var tokenIterator = new AsyncLATokenIterator(inputSource.AsAsync()))
{
result = await parser.ParseInputAsync(tokenIterator, debugger);
}
parseTime.Stop();
var timeElapsed = string.Format("{0} ms", parseTime.Elapsed.TotalMilliseconds);
debugger.WriteFinalToken(
string.Format("Accept ({0}): ", timeElapsed),
string.Format("Error while parsing ({0}): ", timeElapsed),
result);
}
private static async Task MainAsync(string[] args, CancellationToken cancellationToken)
{
var grammar = new Grammar(
Symbols,
PG(Derivation.None,
P(S.Ast, S.Syntax),
PSL(S.Syntax, S.Rule),
PSL(S.Syntax, S.Rule, S.Syntax),
PR(S.Rule, (pLHS, pRHS) => Tuple.Create(pRHS[0].Content, pRHS[2]), S.RuleName, S.DefineOp, S.Clauses, S.RuleEnd)
),
PG(Derivation.LeftMost,
PSL(S.Clauses, S.Terms),
PSL(S.Clauses, S.Terms, S.OrOp, S.Clauses),
PL(S.Terms, MakeMetaTerm, S.Term),
PL(S.Terms, MakeMetaTerm, S.Term, S.TermSep, S.Terms)
),
PG(Derivation.None,
P(S.Term, S.Literal),
P(S.Term, S.RuleName),
PR(S.Term, (pLHS, pRHS) => MakeMetaTerm(null))
),
PG(Derivation.LeftMost,
PR(S.RuleName, MakeQuotedString, S.RuleLiteralLeft, S.RuleCharacters, S.RuleLiteralRight),
PR(S.RuleCharacters, MakeQuotedString, S.RuleCharacter),
PR(S.RuleCharacters, MakeQuotedString, S.RuleCharacter, S.RuleCharacters)
),
PG(Derivation.LeftMost,
PR(S.Literal, MakeQuotedString, S.DoubleQuote, S.TextInDoubleQuotes, S.DoubleQuote),
PR(S.TextInDoubleQuotes, MakeQuotedString, S.CharacterInDoubleQuotes),
PR(S.TextInDoubleQuotes, MakeQuotedString, S.CharacterInDoubleQuotes, S.TextInDoubleQuotes)
),
PG(Derivation.LeftMost,
PR(S.Literal, MakeQuotedString, S.SingleQuote, S.TextInSingleQuotes, S.SingleQuote),
PR(S.TextInSingleQuotes, MakeQuotedString, S.CharacterInSingleQuotes),
PR(S.TextInSingleQuotes, MakeQuotedString, S.CharacterInSingleQuotes, S.TextInSingleQuotes)
)
);
// generate the parse table
var parser = new Parser(grammar);
var debugger = new Debug(parser, Console.Write, Console.Error.Write);
var parseTime = System.Diagnostics.Stopwatch.StartNew();
Func <string, Regex> compileRegex =
p => new Regex("^" + (p.StartsWith("[") || p.Length <= 2
? p
: (p.Substring(0, 2) + "(" + p.Substring(2) + "|$)")),
RegexOptions.CultureInvariant |
RegexOptions.ExplicitCapture);
Func <S, string, string, Tuple <int, Regex, string> > triple =
(pSymbol, pPattern, pState) => Tuple.Create((int)pSymbol, compileRegex(pPattern), pState);
Func <S, string, Tuple <int, Regex, string> > pair =
(pSymbol, pPattern) => triple(pSymbol, pPattern, null);
const string doubleQuoteState = "double";
const string singleQuoteState = "single";
const string doubleQuoteMarker = @"[""]";
const string singleQuoteMarker = "[']";
const string commonSet = "-A-Za-z0-9_";
const string symbolSet = @"| !#$%&()*+,./:;\\<=>?@[\]^`{|}~";
var lexerTable =
new Dictionary <string, Tuple <int, Regex, string>[]>
{
{
AsyncRegexLexer.RootState, new[]
{
triple(S.DoubleQuote, doubleQuoteMarker, doubleQuoteState),
triple(S.SingleQuote, singleQuoteMarker, singleQuoteState),
pair(S.RuleCharacter, "[" + commonSet + "]"),
pair(S.DefineOp, "::="),
pair(S.OrOp, "[|]"),
pair(S.RuleLiteralLeft, "[<]"),
pair(S.RuleLiteralRight, "[>]"),
pair(S.RuleEnd, "[.]"),
pair(S.TermSep, "[,]"),
triple(S.EOL, @"[\r]?[\n]", AsyncRegexLexer.Ignore),
triple(S.WS, @"[ \t]", AsyncRegexLexer.Ignore)
}
},
{
doubleQuoteState, new[]
{
triple(S.DoubleQuote, doubleQuoteMarker, AsyncRegexLexer.PopState),
pair(S.CharacterInDoubleQuotes, "[" + commonSet + symbolSet + "'" + "]")
}
},
{
singleQuoteState, new[]
{
triple(S.SingleQuote, singleQuoteMarker, AsyncRegexLexer.PopState),
pair(S.CharacterInSingleQuotes, "[" + commonSet + symbolSet + "\"" + "]")
}
}
};
Item result;
using (var charIterator = new AsyncLACharIterator(new StringReader(BNF)))
using (var regexLexer = new AsyncRegexLexer(charIterator, lexerTable))
using (var tokenIterator = new AsyncLATokenIterator(regexLexer))
{
result = await parser.ParseInputAsync(tokenIterator, debugger);
}
parseTime.Stop();
var timeElapsed = string.Format("{0} ms", parseTime.Elapsed.TotalMilliseconds);
debugger.WriteFinalToken(
string.Format("Accept ({0}): ", timeElapsed),
string.Format("Error while parsing ({0}): ", timeElapsed),
result);
}
