public override void Validate(LexerSpec lexerSpec)
{
foreach (var expression in Expressions)
{
expression.Validate(lexerSpec);
}
}
public override RegexSpec Simplify(LexerSpec lexerSpec)
{
// a-b -> !(!a|b)
var expanded = Expressions.Aggregate((exp1, exp2) => new ComplementSpec(new AlternationSpec(new[] { new ComplementSpec(exp1), exp2 })));
return(expanded.Simplify(lexerSpec));
}
public override void Validate(LexerSpec lexerSpec)
{
if (!lexerSpec.Rules.Contains(RuleName))
{
throw new Exception($"Reference to rule '{RuleName}' that isn't in lexer spec.");
}
}
public sealed override RegexSpec Simplify(LexerSpec lexerSpec)
{
var codePoints = InversionListCodePointSet.Empty;
var simplified = new List <RegexSpec>();
foreach (var exp in Expressions.Select(e => e.Simplify(lexerSpec)))
{
var charClass = exp.AsCharClass();
if (charClass != null)
{
// [x]|[y] -> [xy]
codePoints = codePoints.Union(charClass.CodePoints);
}
else
{
simplified.Add(exp);
}
}
if (!codePoints.IsEmpty())
{
simplified.Add(new CharClassSpec(codePoints));
}
// All expressions reduced
if (simplified.Count == 1)
{
return(simplified[0]);
}
return(Expressions.SequenceEqual(simplified) ? this : new AlternationSpec(simplified));
}
public override void Validate(RuleSpec rule, LexerSpec lexer)
{
if (!lexer.Channels.Contains(Channel))
{
throw new Exception($"Rule '{rule.Name}' references channel not in lexer spec: '{Channel}'");
}
}
public override RegexSpec Simplify(LexerSpec lexerSpec)
{
// a&b&c -> !(!a|!b|!c)
var expanded = new ComplementSpec(new AlternationSpec(Expressions.Select(exp => new ComplementSpec(exp))));
return(expanded.Simplify(lexerSpec));
}
public override void Validate(RuleSpec rule, LexerSpec lexer)
{
if (!lexer.Modes.Contains(Mode))
{
throw new Exception($"Rule '{rule.Name}' references mode not in lexer spec: '{Mode}'");
}
}
public void Validate(LexerSpec lexer)
{
var modeReferenceErrors = Modes.Except(lexer.Modes).ToList();
if(modeReferenceErrors.Any())
throw new Exception($"Rule '{Name}' references mode(s) not in lexer spec: '{string.Join("', '", modeReferenceErrors)}'");
ValidateCommands(lexer);
Expression.Validate(lexer);
}
public RuleSpec Simplify(ISet<Mode> modesEntered, LexerSpec lexer)
{
var reducedModes = new HashSet<Mode>(Modes.Intersect(modesEntered));
var simplfiedExpression = Expression.Simplify(lexer);
if(reducedModes.Count >= Modes.Count && Expression == simplfiedExpression) return this;
return new RuleSpec(reducedModes, Name, simplfiedExpression, Commands);
}
public override void Validate(RuleSpec rule, LexerSpec lexer)
{
if(!lexer.Rules.Contains(TokenType))
throw new Exception($"Rule '{rule.Name}' references type not in lexer spec: '{TokenType}'");
if(lexer.Rules[TokenType].IsFragment)
throw new Exception($"Rule '{rule.Name}' references fragment for token type: '{TokenType}'");
}
protected override RegexSpec Simplify(LexerSpec lexerSpec, RegexSpec exp, bool changed)
{
// ~a -> !([^]* a [^]*)
var anyChar = new CharClassSpec(InversionListCodePointSet.All);
var repetition = new RepetitionSpec(anyChar, 0, null);
var concatenation = new ConcatenationSpec(new[] { repetition, exp, repetition });
var expanded = new ComplementSpec(concatenation);
return(expanded.Simplify(lexerSpec));
}
protected override RegexSpec Simplify(LexerSpec lexerSpec, RegexSpec exp, bool changed)
{
// x{1,1} -> x
if (MinRepititions == 1 && MaxRepititions == 1)
{
return(exp);
}
return(changed ? new RepetitionSpec(exp, MinRepititions, MaxRepititions) : this);
}
public void Validate(LexerSpec lexer)
{
var modeReferenceErrors = Modes.Except(lexer.Modes).ToList();
if (modeReferenceErrors.Any())
{
throw new Exception($"Rule '{Name}' references mode(s) not in lexer spec: '{string.Join("', '", modeReferenceErrors)}'");
}
ValidateCommands(lexer);
Expression.Validate(lexer);
}
public override void Validate(RuleSpec rule, LexerSpec lexer)
{
if (!lexer.Rules.Contains(TokenType))
{
throw new Exception($"Rule '{rule.Name}' references type not in lexer spec: '{TokenType}'");
}
if (lexer.Rules[TokenType].IsFragment)
{
throw new Exception($"Rule '{rule.Name}' references fragment for token type: '{TokenType}'");
}
}
public RuleSpec Simplify(ISet <Mode> modesEntered, LexerSpec lexer)
{
var reducedModes = new HashSet <Mode>(Modes.Intersect(modesEntered));
var simplfiedExpression = Expression.Simplify(lexer);
if (reducedModes.Count >= Modes.Count && Expression == simplfiedExpression)
{
return(this);
}
return(new RuleSpec(reducedModes, Name, simplfiedExpression, Commands));
}
protected override RegexSpec Simplify(LexerSpec lexerSpec, RegexSpec exp, bool changed)
{
// ![x] -> [^x]
var charClass = Expression.Simplify(lexerSpec).AsCharClass();
if (charClass != null)
{
return(new CharClassSpec(charClass.CodePoints.Complement()));
}
// !!x -> x
var complement = exp as ComplementSpec;
if (complement != null)
{
return(complement.Expression);
}
return(changed ? new ComplementSpec(exp) : this);
}
public override void Validate(LexerSpec lexerSpec)
{
Expression.Validate(lexerSpec);
}
public override void Validate(LexerSpec lexerSpec)
{
Expression.Validate(lexerSpec);
}
public override RegexSpec Simplify(LexerSpec lexerSpec)
{
var simplified = Expressions.Select(e => e.Simplify(lexerSpec)).ToList();
return(Expressions.SequenceEqual(simplified) ? this : new ConcatenationSpec(simplified));
}
public virtual void Validate(RuleSpec rule, LexerSpec lexer)
{
}
protected abstract RegexSpec Simplify(LexerSpec lexerSpec, RegexSpec exp, bool changed);
public virtual void Validate(LexerSpec lexerSpec)
{
}
static ThrowAway()
{
var newline = new AlternationSpec(new RegexSpec[] { "\r", "\n", "\u2028", "\u2029", "\u000B", "\u000C", "\u0085", "\r\n" });
var whitespace = new AlternationSpec(new RegexSpec[] { newline, " ", "\t", "\f", "\v" });
var digit = Class('0', '9');
var hexDigit = new AlternationSpec(new[] { digit, Class('a', 'f'), Class('A', 'F') });
var charClassChar = new CharClassSpec(new InversionListCodePointSet('\\')
.Union(new InversionListCodePointSet('-'))
.Union(new InversionListCodePointSet(']')).Complement());
var defaultChannel = new Channel("Default");
var whiteSpaceChannel = new Channel("WhiteSpace");
var initial = new Mode("Default");
var startCharClass = new Mode("StartCharacterClass");
var charClass = new Mode("CharacterClass");
SpecLexerSpec = new LexerSpec("SpecLexer", "Adamant.CompilerCompiler.Lex.SpecParsing", new[]
{
// Comments
new RuleSpec(initial, "Comment", ("/*" + ~R("*/")) | ("//" + ~newline), Command.SetChannel(whiteSpaceChannel)),
new RuleSpec(initial, "WhiteSpace", whitespace.RepeatAtLeast(1), Command.SetChannel(whiteSpaceChannel)),
// Commands
new RuleSpec(initial, "Mode", "@mode"),
new RuleSpec(initial, "PushMode", "@pushMode"),
new RuleSpec(initial, "PopMode", "@popMode"),
new RuleSpec(initial, "Skip", "@skip"),
new RuleSpec(initial, "More", "@more"),
new RuleSpec(initial, "Type", "@type"),
new RuleSpec(initial, "Channel", "@channel"),
new RuleSpec(initial, "Error", "@error"),
new RuleSpec(initial, "Capture", "@capture"),
new RuleSpec(initial, "Decode", "@decode"),
new RuleSpec(initial, "Text", "@text"),
new RuleSpec(initial, "Action", "<%" + ~R("%>"), Command.Capture),
// Keywords
new RuleSpec(initial, "Lexer", "@lexer"),
new RuleSpec(initial, "Namespace", "@namespace"),
new RuleSpec(initial, "Modes", "@modes"),
new RuleSpec(initial, "Channels", "@channels"),
new RuleSpec(initial, "InvalidKeyword", "@" + new RuleReferenceSpec("Identifier"), Command.Capture, Command.FlagError),
// Expression Operators
new RuleSpec(initial, "Definition", ":"),
new RuleSpec(initial, "Alternation", "|"),
new RuleSpec(initial, "BeginCharClass", "[", Command.Capture, Command.PushMode(startCharClass)),
new RuleSpec(initial, "AnyChar", "."),
new RuleSpec(initial, "Optional", "?"),
new RuleSpec(initial, "Complement", "!"),
new RuleSpec(initial, "Repetition", "*"),
new RuleSpec(initial, "Intersection", "&"),
new RuleSpec(initial, "Subtraction", "-"),
new RuleSpec(initial, "Upto", "~"),
new RuleSpec(initial, "BeginGroup", "("),
new RuleSpec(initial, "EndGroup", ")"),
new RuleSpec(initial, "BeginningOfLine", "^"),
new RuleSpec(initial, "EndOfLine", "$"),
new RuleSpec(initial, "BeginQuantifier", "{"),
new RuleSpec(initial, "EndQuantifier", "}"),
new RuleSpec(initial, "BeginCommands", "->"),
new RuleSpec(initial, "Terminator", ";"),
new RuleSpec(initial, "Comma", ","),
// Terminals
new RuleSpec(initial, "Number", "0" | (Class('1', '9') + digit), Command.Capture),
new RuleSpec(initial, "Identifier", (Class('a', 'z') | Class('A', 'Z')) + (Class('a', 'z') | Class('A', 'Z') | digit).Repeat(), Command.Capture),
new RuleSpec(initial, "Literal", R("\"") + new RuleReferenceSpec("literalChar").RepeatAtLeast(1) + R("\"") | new RuleReferenceSpec("escapeChar"), Command.Capture),
new RuleSpec(initial, "literalChar", new RuleReferenceSpec("escapeChar") | !(R("\\") | "\"")),
new RuleSpec(initial, "Category", R("\\R") | "\\s" | "\\d"),
// Fragments
new RuleSpec(initial, "escapeChar",
R("\\t")
| R("\\n")
| R("\\r")
| R("\\b")
| R("\\f")
| R("\\0")
| R("\\a")
| R("\\v")
| R("\\\"")
| R("\\{")
| R("\\'")
| R("\\\\")
| (R("\\x") + hexDigit.Repeat(2))
| (R("\\u") + hexDigit.Repeat(4))
| (R("\\U") + hexDigit.Repeat(6))
| (R("\\u{") + hexDigit.Repeat(1, 6) + R("}"))),
// Fallback
new RuleSpec(initial, "UnexpectedCodePoint", new CharClassSpec(InversionListCodePointSet.All), Command.Capture, Command.FlagError),
// Character Classes
new RuleSpec(startCharClass, "NegateCharClass", "^", Command.SetMode(charClass)),
new RuleSpec(charClass, "Char", new RuleReferenceSpec("escapeChar") | charClassChar, Command.Capture, Command.SetMode(charClass)),
new RuleSpec(charClass, "EscapeDash", R("\\-"), Command.Text("-"), Command.SetType("Char"), Command.SetMode(charClass)),
new RuleSpec(charClass, "EscapeRightBracket", R("\\]"), Command.Text("-"), Command.SetType("Char"), Command.SetMode(charClass)),
new RuleSpec(charClass, "CharRange", "-", Command.SetMode(charClass)),
new RuleSpec(charClass, "EndCharClass", "]", Command.PopMode),
},
new[] { defaultChannel, whiteSpaceChannel },
defaultChannel,
new[] { initial, startCharClass, charClass },
initial);
}
public override void Validate(RuleSpec rule, LexerSpec lexer)
{
if(!lexer.Modes.Contains(Mode))
throw new Exception($"Rule '{rule.Name}' references mode not in lexer spec: '{Mode}'");
}
public override void Validate(RuleSpec rule, LexerSpec lexer)
{
if(!lexer.Channels.Contains(Channel))
throw new Exception($"Rule '{rule.Name}' references channel not in lexer spec: '{Channel}'");
}
public override RegexSpec Simplify(LexerSpec lexerSpec)
{
return(lexerSpec.Rules[RuleName].Expression.Simplify(lexerSpec));
}
private void ValidateCommands(LexerSpec lexer)
{
if (IsFragment && Commands.Any())
{
throw new Exception($"Rule '{Name}' is a fragment, but has commands. This is not allowed");
}
if (Commands.Count(c => c == Command.Skip || c == Command.More || c is SetTypeCommand) > 1)
{
throw new Exception($"Rule '{Name}', only one of @skip, @more or @type command is allowed per rule");
}
if (Commands.Count(c => c == Command.Capture || c is DecodeCommand || c is TextCommand) > 1)
{
throw new Exception($"Rule '{Name}', only one of @capture, @decode or @text command is allowed per rule");
}
if (Commands.Contains(Command.Skip) && Commands.Contains(Command.FlagError))
{
throw new Exception($"Rule '{Name}', skipped rules can't be marked @error");
}
if (Commands.Contains(Command.Skip) && Commands.Contains(Command.Capture))
{
throw new Exception($"Rule '{Name}', skipped rules can't be marked @capture");
}
if (Commands.Contains(Command.Skip) && Commands.Any(c => c is DecodeCommand))
{
throw new Exception($"Rule '{Name}', skipped rules can't be marked @decode");
}
if (Commands.Contains(Command.Skip) && Commands.Any(c => c is DecodeCommand))
{
throw new Exception($"Rule '{Name}', skipped rules can't be marked @text");
}
if (Commands.Contains(Command.Skip) && Commands.Any(c => c is SetChannelCommand))
{
throw new Exception($"Rule '{Name}', skipped rules can't be marked with a @channel");
}
if (Commands.Contains(Command.More) && Commands.Contains(Command.FlagError))
{
throw new Exception($"Rule '{Name}', more rules can't be marked @error");
}
if (Commands.Contains(Command.More) && Commands.Any(c => c is SetChannelCommand))
{
throw new Exception($"Rule '{Name}', more rules can't be marked with a @channel");
}
if (Commands.OfType <SetChannelCommand>().Count() > 1)
{
throw new Exception($"Rule '{Name}', only one @channel command is allowed per rule");
}
if (Commands.Count(c => c == Command.FlagError) > 1)
{
throw new Exception($"Rule '{Name}', only one @error command is allowed per rule");
}
if (Commands.Reverse().Skip(1).Any(c => c is CodeActionCommand))
{
throw new Exception($"Rule '{Name}', there can only be one code action per rule, and it must be the last command");
}
foreach (var command in Commands)
{
command.Validate(this, lexer);
}
}
public override RegexSpec Simplify(LexerSpec lexerSpec)
{
var simplifiedExpression = Expression.Simplify(lexerSpec);
return Simplify(lexerSpec, simplifiedExpression, simplifiedExpression != Expression);
}
public override RegexSpec Simplify(LexerSpec lexerSpec)
{
var simplifiedExpression = Expression.Simplify(lexerSpec);
return(Simplify(lexerSpec, simplifiedExpression, simplifiedExpression != Expression));
}
protected abstract RegexSpec Simplify(LexerSpec lexerSpec, RegexSpec exp, bool changed);
private void ValidateCommands(LexerSpec lexer)
{
if(IsFragment && Commands.Any())
throw new Exception($"Rule '{Name}' is a fragment, but has commands. This is not allowed");
if(Commands.Count(c => c == Command.Skip || c == Command.More || c is SetTypeCommand) > 1)
throw new Exception($"Rule '{Name}', only one of @skip, @more or @type command is allowed per rule");
if(Commands.Count(c => c == Command.Capture || c is DecodeCommand || c is TextCommand) > 1)
throw new Exception($"Rule '{Name}', only one of @capture, @decode or @text command is allowed per rule");
if(Commands.Contains(Command.Skip) && Commands.Contains(Command.FlagError))
throw new Exception($"Rule '{Name}', skipped rules can't be marked @error");
if(Commands.Contains(Command.Skip) && Commands.Contains(Command.Capture))
throw new Exception($"Rule '{Name}', skipped rules can't be marked @capture");
if(Commands.Contains(Command.Skip) && Commands.Any(c => c is DecodeCommand))
throw new Exception($"Rule '{Name}', skipped rules can't be marked @decode");
if(Commands.Contains(Command.Skip) && Commands.Any(c => c is DecodeCommand))
throw new Exception($"Rule '{Name}', skipped rules can't be marked @text");
if(Commands.Contains(Command.Skip) && Commands.Any(c => c is SetChannelCommand))
throw new Exception($"Rule '{Name}', skipped rules can't be marked with a @channel");
if(Commands.Contains(Command.More) && Commands.Contains(Command.FlagError))
throw new Exception($"Rule '{Name}', more rules can't be marked @error");
if(Commands.Contains(Command.More) && Commands.Any(c => c is SetChannelCommand))
throw new Exception($"Rule '{Name}', more rules can't be marked with a @channel");
if(Commands.OfType<SetChannelCommand>().Count() > 1)
throw new Exception($"Rule '{Name}', only one @channel command is allowed per rule");
if(Commands.Count(c => c == Command.FlagError) > 1)
throw new Exception($"Rule '{Name}', only one @error command is allowed per rule");
if(Commands.Reverse().Skip(1).Any(c => c is CodeActionCommand))
throw new Exception($"Rule '{Name}', there can only be one code action per rule, and it must be the last command");
foreach(var command in Commands)
command.Validate(this, lexer);
}
public override void Validate(LexerSpec lexerSpec)
{
foreach(var expression in Expressions)
expression.Validate(lexerSpec);
}
public virtual RegexSpec Simplify(LexerSpec lexerSpec)
{
return(this);
}
