internal static XbnfProduction Parse(ParseContext pc)
{
var result = new XbnfProduction();
pc.TrySkipCCommentsAndWhiteSpace();
// read identifier
result.Name = ParseIdentifier(pc);
// read attributes
if ('<' == pc.Current)
{
pc.Advance();
while (-1 != pc.Current && '>' != pc.Current)
{
result.Attributes.Add(XbnfAttribute.Parse(pc));
pc.TrySkipCCommentsAndWhiteSpace();
pc.Expecting('>', ',');
if (',' == pc.Current)
{
pc.Advance();
}
}
pc.Expecting('>');
pc.Advance();
}
pc.TrySkipCCommentsAndWhiteSpace();
pc.Expecting(';', '=');
if ('=' == pc.Current)
{
pc.Advance();
result.Expression = XbnfExpression.Parse(pc);
}
pc.Expecting(';');
pc.Advance();
return(result);
}
static RegexExpression _ParseModifier(RegexExpression expr, ParseContext pc)
{
var line = pc.Line;
var column = pc.Column;
var position = pc.Position;
switch (pc.Current)
{
case '*':
expr = new RegexRepeatExpression(expr);
expr.SetLocation(line, column, position);
pc.Advance();
break;
case '+':
expr = new RegexRepeatExpression(expr, 1);
expr.SetLocation(line, column, position);
pc.Advance();
break;
case '?':
expr = new RegexOptionalExpression(expr);
expr.SetLocation(line, column, position);
pc.Advance();
break;
case '{':
pc.Advance();
pc.TrySkipWhiteSpace();
pc.Expecting('0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ',', '}');
var min = -1;
var max = -1;
if (',' != pc.Current && '}' != pc.Current)
{
var l = pc.CaptureBuffer.Length;
pc.TryReadDigits();
min = int.Parse(pc.GetCapture(l));
pc.TrySkipWhiteSpace();
}
if (',' == pc.Current)
{
pc.Advance();
pc.TrySkipWhiteSpace();
pc.Expecting('0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '}');
if ('}' != pc.Current)
{
var l = pc.CaptureBuffer.Length;
pc.TryReadDigits();
max = int.Parse(pc.GetCapture(l));
pc.TrySkipWhiteSpace();
}
}
pc.Expecting('}');
pc.Advance();
expr = new RegexRepeatExpression(expr, min, max);
expr.SetLocation(line, column, position);
break;
}
return(expr);
}
static void _ParseAttribute(EbnfDocument doc, string id, EbnfProduction prod, ParseContext pc)
{
pc.TrySkipCCommentsAndWhiteSpace();
var attrid = _ParseIdentifier(pc);
pc.TrySkipCCommentsAndWhiteSpace();
pc.Expecting('=', '>', ',');
object val = true;
if ('=' == pc.Current)
{
pc.Advance();
val = pc.ParseJsonValue();
}
pc.Expecting(',', '>');
prod.Attributes[attrid] = val;
pc.TrySkipCCommentsAndWhiteSpace();
}
static void _ParseAttributes(EbnfDocument doc, string id, EbnfProduction prod, ParseContext pc)
{
pc.TrySkipCCommentsAndWhiteSpace();
pc.Expecting('<');
pc.Advance();
while (-1 != pc.Current && '>' != pc.Current)
{
_ParseAttribute(doc, id, prod, pc);
pc.TrySkipCCommentsAndWhiteSpace();
pc.Expecting(',', '>');
if (',' == pc.Current)
{
pc.Advance();
}
}
pc.Expecting('>');
pc.Advance();
pc.TrySkipCCommentsAndWhiteSpace();
}
static void _ParseProduction(EbnfDocument doc, ParseContext pc)
{
pc.TrySkipCCommentsAndWhiteSpace();
var line = pc.Line;
var column = pc.Column;
var position = pc.Position;
var id = _ParseIdentifier(pc);
pc.TrySkipCCommentsAndWhiteSpace();
EbnfProduction prod = null;
if (!doc.Productions.TryGetValue(id, out prod))
{
prod = new EbnfProduction();
doc.Productions.Add(id, prod);
}
if ('<' == pc.Current)
{
_ParseAttributes(doc, id, prod, pc);
pc.TrySkipCCommentsAndWhiteSpace();
}
pc.Expecting('=');
pc.Advance();
pc.Expecting();
var expr = _ParseExpression(doc, pc);
pc.TrySkipCCommentsAndWhiteSpace();
pc.Expecting(';');
pc.Advance();
pc.TrySkipCCommentsAndWhiteSpace();
// transform this into an OrExpression with the previous
if (null != prod.Expression)
{
prod.Expression = new EbnfOrExpression(prod.Expression, expr);
}
else
{
prod.Expression = expr;
}
prod.SetLocationInfo(line, column, position);
}
static string _ParseIdentifier(ParseContext pc)
{
pc.TrySkipCCommentsAndWhiteSpace();
if (-1 == pc.Current)
{
pc.Expecting();
return(null);
}
var l = pc.CaptureBuffer.Length;
if ('_' != pc.Current && !char.IsLetter((char)pc.Current))
{
pc.Expecting("ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvwxyz".ToCharArray().Convert <int>().ToArray());
}
pc.CaptureCurrent();
while (-1 != pc.Advance() && ('_' == pc.Current || '-' == pc.Current || char.IsLetterOrDigit((char)pc.Current)))
{
pc.CaptureCurrent();
}
pc.TrySkipCCommentsAndWhiteSpace();
return(pc.GetCapture(l));
}
static CfgRule _Parse(ParseContext pc)
{
var result = new CfgRule();
pc.TrySkipWhiteSpace();
pc.ClearCapture();
pc.TryReadUntil(false, ' ', '\t', '\r', '\n', '\f', '\v', '-');
result.Left = pc.Capture;
pc.TrySkipWhiteSpace();
pc.Expecting('-');
pc.Advance();
pc.Expecting('>');
pc.Advance();
while (-1 != pc.Current && '\n' != pc.Current)
{
pc.TrySkipWhiteSpace();
pc.ClearCapture();
pc.TryReadUntil(false, ' ', '\t', '\r', '\n', '\f', '\v');
result.Right.Add(pc.Capture);
}
return(result);
}
internal static XbnfAttribute Parse(ParseContext pc)
{
pc.TrySkipCCommentsAndWhiteSpace();
var attr = new XbnfAttribute();
attr.SetLocation(pc.Line, pc.Column, pc.Position);
attr.Name = ParseIdentifier(pc);
pc.TrySkipCCommentsAndWhiteSpace();
pc.Expecting(',', '=', ',', '>');
if ('=' == pc.Current)
{
pc.Advance();
attr.Value = pc.ParseJsonValue();
}
else
{
attr.Value = true;
}
return(attr);
}
internal static LexAttribute Parse(ParseContext pc)
{
SkipCommentsAndWhitespace(pc);
var attr = new LexAttribute();
attr.SetLocation(pc.Line, pc.Column, pc.Position);
attr.Name = _ParseAttrName(pc);
SkipCommentsAndWhitespace(pc);
pc.Expecting(',', '=', ',', '>', '\n');
if ('=' == pc.Current)
{
pc.Advance();
attr.Value = pc.ParseJsonValue();
}
else
{
attr.Value = true;
}
return(attr);
}
static EbnfExpression _ParseExpression(EbnfDocument doc, ParseContext pc)
{
EbnfExpression current = null;
EbnfExpression e;
long position;
int line;
int column;
pc.TrySkipCCommentsAndWhiteSpace();
position = pc.Position; line = pc.Line; column = pc.Column;
while (-1 != pc.Current && ']' != pc.Current && ')' != pc.Current && '}' != pc.Current && ';' != pc.Current)
{
pc.TrySkipCCommentsAndWhiteSpace();
position = pc.Position; line = pc.Line; column = pc.Column;
switch (pc.Current)
{
case '|':
pc.Advance();
current = new EbnfOrExpression(current, _ParseExpression(doc, pc));
current.SetLocationInfo(line, column, position);
break;
case '(':
pc.Advance();
e = _ParseExpression(doc, pc);
current.SetLocationInfo(line, column, position);
pc.Expecting(')');
pc.Advance();
e.SetLocationInfo(line, column, position);
if (null == current)
{
current = e;
}
else
{
current = new EbnfConcatExpression(current, e);
}
break;
case '[':
pc.Advance();
e = new EbnfOptionalExpression(_ParseExpression(doc, pc));
e.SetLocationInfo(line, column, position);
pc.TrySkipCCommentsAndWhiteSpace();
pc.Expecting(']');
pc.Advance();
if (null == current)
{
current = e;
}
else
{
current = new EbnfConcatExpression(current, e);
}
break;
case '{':
pc.Advance();
e = new EbnfRepeatExpression(_ParseExpression(doc, pc));
e.SetLocationInfo(line, column, position);
pc.TrySkipCCommentsAndWhiteSpace();
pc.Expecting('}');
pc.Advance();
if ('+' == pc.Current)
{
pc.Advance();
((EbnfRepeatExpression)e).IsOptional = false;
}
if (null == current)
{
current = e;
}
else
{
current = new EbnfConcatExpression(current, e);
}
break;
case '\"':
e = new EbnfLiteralExpression(pc.ParseJsonString());
if (null == current)
{
current = e;
}
else
{
current = new EbnfConcatExpression(current, e);
}
e.SetLocationInfo(line, column, position);
break;
case '\'':
pc.Advance();
pc.ClearCapture();
pc.TryReadUntil('\'', '\\', false);
pc.Expecting('\'');
pc.Advance();
e = new EbnfRegexExpression(pc.Capture);
if (null == current)
{
current = e;
}
else
{
current = new EbnfConcatExpression(current, e);
}
e.SetLocationInfo(line, column, position);
break;
case ';':
case ']':
case ')':
case '}':
return(current);
default:
e = new EbnfRefExpression(_ParseIdentifier(pc));
if (null == current)
{
current = e;
}
else
{
current = new EbnfConcatExpression(current, e);
}
e.SetLocationInfo(line, column, position);
break;
}
}
pc.TrySkipCCommentsAndWhiteSpace();
return(current);
}
static CharFA <TAccept> _Parse(ParseContext pc, TAccept accept)
{
CharFA <TAccept> result = new CharFA <TAccept>(true, accept);
CharFA <TAccept> f, next;
int ch;
pc.EnsureStarted();
var current = result;
while (true)
{
switch (pc.Current)
{
case -1:
return(result);
case '.':
pc.Advance();
f = current.FirstAcceptingState as CharFA <TAccept>;
current = Set(new CharRange[] { new CharRange(char.MinValue, char.MaxValue) }, accept);
switch (pc.Current)
{
case '*':
current = Kleene(current, accept);
pc.Advance();
break;
case '+':
current = Repeat(current, accept);
pc.Advance();
break;
case '?':
current = Optional(current, accept);
pc.Advance();
break;
}
f.IsAccepting = false;
f.EpsilonTransitions.Add(current);
break;
case '\\':
if (-1 != (ch = _ParseEscape(pc)))
{
next = null;
switch (pc.Current)
{
case '*':
next = new CharFA <TAccept>();
next.Transitions.Add((char)ch, new CharFA <TAccept>(true, accept));
next = Kleene(next, accept);
pc.Advance();
break;
case '+':
next = new CharFA <TAccept>();
next.Transitions.Add((char)ch, new CharFA <TAccept>(true, accept));
next = Repeat(next, accept);
pc.Advance();
break;
case '?':
next = new CharFA <TAccept>();
next.Transitions.Add((char)ch, new CharFA <TAccept>(true, accept));
next = Optional(next, accept);
pc.Advance();
break;
default:
current = current.FirstAcceptingState as CharFA <TAccept>;
current.IsAccepting = false;
current.Transitions.Add((char)ch, new CharFA <TAccept>(true, accept));
break;
}
if (null != next)
{
current = current.FirstAcceptingState as CharFA <TAccept>;
current.IsAccepting = false;
current.EpsilonTransitions.Add(next);
current = next;
}
}
else
{
pc.Expecting(); // throw an error
return(null); // doesn't execute
}
break;
case ')':
return(result);
case '(':
pc.Advance();
pc.Expecting();
f = current.FirstAcceptingState as CharFA <TAccept>;
current = _Parse(pc, accept);
pc.Expecting(')');
pc.Advance();
switch (pc.Current)
{
case '*':
current = Kleene(current, accept);
pc.Advance();
break;
case '+':
current = Repeat(current, accept);
pc.Advance();
break;
case '?':
current = Optional(current, accept);
pc.Advance();
break;
}
var ff = f.FirstAcceptingState;
ff.EpsilonTransitions.Add(current);
ff.IsAccepting = false;
break;
case '|':
if (-1 != pc.Advance())
{
current = _Parse(pc, accept);
result = Or(new CharFA <TAccept>[] { result as CharFA <TAccept>, current as CharFA <TAccept> }, accept);
}
else
{
current = current.FirstAcceptingState as CharFA <TAccept>;
result = Optional(result, accept);
}
break;
case '[':
pc.ClearCapture();
pc.Advance();
pc.Expecting();
bool not = false;
if ('^' == pc.Current)
{
not = true;
pc.Advance();
pc.Expecting();
}
pc.TryReadUntil(']', '\\', false);
pc.Expecting(']');
pc.Advance();
var r = (!not && "." == pc.Capture) ?
new CharRange[] { new CharRange(char.MinValue, char.MaxValue) } :
_ParseRanges(pc.Capture, true);
if (not)
{
r = CharRange.NotRanges(r);
}
f = current.FirstAcceptingState as CharFA <TAccept>;
current = Set(r, accept);
switch (pc.Current)
{
case '*':
current = Kleene(current, accept);
pc.Advance();
break;
case '+':
current = Repeat(current, accept);
pc.Advance();
break;
case '?':
current = Optional(current, accept);
pc.Advance();
break;
}
f.IsAccepting = false;
f.EpsilonTransitions.Add(current);
break;
default:
ch = pc.Current;
pc.Advance();
next = null;
switch (pc.Current)
{
case '*':
next = new CharFA <TAccept>();
next.Transitions.Add((char)ch, new CharFA <TAccept>(true, accept));
next = Kleene(next, accept);
pc.Advance();
break;
case '+':
next = new CharFA <TAccept>();
next.Transitions.Add((char)ch, new CharFA <TAccept>(true, accept));
next = Repeat(next, accept);
pc.Advance();
break;
case '?':
next = new CharFA <TAccept>();
next.Transitions.Add((char)ch, new CharFA <TAccept>(true, accept));
next = Optional(next, accept);
pc.Advance();
break;
default:
current = current.FirstAcceptingState as CharFA <TAccept>;
current.IsAccepting = false;
current.Transitions.Add((char)ch, new CharFA <TAccept>(true, accept));
break;
}
if (null != next)
{
current = current.FirstAcceptingState as CharFA <TAccept>;
current.IsAccepting = false;
current.EpsilonTransitions.Add(next);
current = next;
}
break;
}
}
}
internal static RegexExpression Parse(ParseContext pc)
{
RegexExpression result = null, next = null;
int ich;
pc.EnsureStarted();
var line = pc.Line;
var column = pc.Column;
var position = pc.Position;
while (true)
{
switch (pc.Current)
{
case -1:
return(result);
case '.':
var nset = new RegexCharsetExpression(new RegexCharsetEntry[] { new RegexCharsetRangeEntry(char.MinValue, char.MaxValue) }, false);
nset.SetLocation(line, column, position);
if (null == result)
{
result = nset;
}
else
{
result = new RegexConcatExpression(result, nset);
result.SetLocation(line, column, position);
}
pc.Advance();
result = _ParseModifier(result, pc);
line = pc.Line;
column = pc.Column;
position = pc.Position;
break;
case '\\':
if (-1 != (ich = _ParseEscape(pc)))
{
next = new RegexLiteralExpression((char)ich);
next.SetLocation(line, column, position);
next = _ParseModifier(next, pc);
if (null != result)
{
result = new RegexConcatExpression(result, next);
result.SetLocation(line, column, position);
}
else
{
result = next;
}
}
else
{
pc.Expecting(); // throw an error
return(null); // doesn't execute
}
line = pc.Line;
column = pc.Column;
position = pc.Position;
break;
case ')':
return(result);
case '(':
pc.Advance();
pc.Expecting();
next = Parse(pc);
pc.Expecting(')');
pc.Advance();
next = _ParseModifier(next, pc);
if (null == result)
{
result = next;
}
else
{
result = new RegexConcatExpression(result, next);
result.SetLocation(line, column, position);
}
line = pc.Line;
column = pc.Column;
position = pc.Position;
break;
case '|':
if (-1 != pc.Advance())
{
next = Parse(pc);
result = new RegexOrExpression(result, next);
result.SetLocation(line, column, position);
}
else
{
result = new RegexOrExpression(result, null);
result.SetLocation(line, column, position);
}
line = pc.Line;
column = pc.Column;
position = pc.Position;
break;
case '[':
pc.ClearCapture();
pc.Advance();
pc.Expecting();
bool not = false;
if ('^' == pc.Current)
{
not = true;
pc.Advance();
pc.Expecting();
}
var ranges = _ParseRanges(pc);
if (ranges.Count == 0)
{
System.Diagnostics.Debugger.Break();
}
pc.Expecting(']');
pc.Advance();
next = new RegexCharsetExpression(ranges, not);
next.SetLocation(line, column, position);
next = _ParseModifier(next, pc);
if (null == result)
{
result = next;
}
else
{
result = new RegexConcatExpression(result, next);
result.SetLocation(pc.Line, pc.Column, pc.Position);
}
line = pc.Line;
column = pc.Column;
position = pc.Position;
break;
default:
ich = pc.Current;
next = new RegexLiteralExpression((char)ich);
next.SetLocation(line, column, position);
pc.Advance();
next = _ParseModifier(next, pc);
if (null == result)
{
result = next;
}
else
{
result = new RegexConcatExpression(result, next);
result.SetLocation(line, column, position);
}
line = pc.Line;
column = pc.Column;
position = pc.Position;
break;
}
}
}
static IList <RegexCharsetEntry> _ParseRanges(ParseContext pc)
{
pc.EnsureStarted();
var result = new List <RegexCharsetEntry>();
RegexCharsetEntry next = null;
bool readDash = false;
while (-1 != pc.Current && ']' != pc.Current)
{
switch (pc.Current)
{
case '[': // char class
if (null != next)
{
result.Add(next);
if (readDash)
{
result.Add(new RegexCharsetCharEntry('-'));
}
result.Add(new RegexCharsetCharEntry('-'));
}
pc.Advance();
pc.Expecting(':');
pc.Advance();
var l = pc.CaptureBuffer.Length;
pc.TryReadUntil(':', false);
var n = pc.GetCapture(l);
pc.Advance();
pc.Expecting(']');
pc.Advance();
result.Add(new RegexCharsetClassEntry(n));
readDash = false;
next = null;
break;
case '\\':
//pc.Advance();
//pc.Expecting();
var ch = (char)_ParseEscape(pc);
if (null == next)
{
next = new RegexCharsetCharEntry(ch);
}
else
{
if (readDash)
{
result.Add(new RegexCharsetRangeEntry(((RegexCharsetCharEntry)next).Value, ch));
next = null;
readDash = false;
}
else
{
result.Add(next);
next = new RegexCharsetCharEntry(ch);
}
}
break;
case '-':
pc.Advance();
if (null == next)
{
next = new RegexCharsetCharEntry('-');
readDash = false;
}
else
{
if (readDash)
{
result.Add(next);
}
readDash = true;
}
break;
default:
if (null == next)
{
next = new RegexCharsetCharEntry((char)pc.Current);
}
else
{
if (readDash)
{
result.Add(new RegexCharsetRangeEntry(((RegexCharsetCharEntry)next).Value, (char)pc.Current));
next = null;
readDash = false;
}
else
{
result.Add(next);
next = new RegexCharsetCharEntry((char)pc.Current);
}
}
pc.Advance();
break;
}
}
if (null != next)
{
result.Add(next);
if (readDash)
{
next = new RegexCharsetCharEntry('-');
result.Add(next);
}
}
return(result);
}
internal static LexDocument Parse(ParseContext pc)
{
var result = new LexDocument();
while (-1 != pc.Current)
{
var line = pc.Line;
var column = pc.Column;
var position = pc.Position;
LexNode.SkipCommentsAndWhitespace(pc);
while ('\n' == pc.Current)
{
pc.Advance();
LexNode.SkipCommentsAndWhitespace(pc);
}
var id = LexNode.ParseIdentifier(pc);
if (string.IsNullOrEmpty(id))
{
pc.Advance();
LexNode.SkipCommentsAndWhitespace(pc);
continue;
}
LexNode.SkipCommentsAndWhitespace(pc);
pc.Expecting(':', '-', '=');
if (':' == pc.Current) // attribute set
{
pc.Advance();
var d = new LexAttributeList();
while (-1 != pc.Current && '\n' != pc.Current)
{
var attr = LexAttribute.Parse(pc);
d.Add(attr);
LexNode.SkipCommentsAndWhitespace(pc);
pc.Expecting('\n', ',', -1);
if (',' == pc.Current)
{
pc.Advance();
}
}
result.AttributeSets.Add(id, d);
LexNode.SkipCommentsAndWhitespace(pc);
}
else if ('=' == pc.Current)
{
pc.Advance();
LexNode.SkipCommentsAndWhitespace(pc);
pc.Expecting('\'');
pc.Advance();
var l = pc.CaptureBuffer.Length;
pc.TryReadUntil('\'', '\\', false);
pc.Expecting('\'');
pc.Advance();
var rx = pc.GetCapture(l);
// make sure to capture the line numbers properly:
var rpc = ParseContext.Create(rx);
rpc.EnsureStarted();
rpc.SetLocation(pc.Line, pc.Column, pc.Position);
var rule = new LexRule(id, RegexExpression.Parse(rpc));
rule.SetLocation(line, column, position);
result.Rules.Add(rule);
}
else if ('-' == pc.Current)
{
pc.TrySkipUntil('\n', true);
}
LexNode.SkipCommentsAndWhitespace(pc);
if ('\n' == pc.Current)
{
pc.Advance();
}
}
return(result);
}
