private int Test <TLetter>(string pattern, bool caseSensitive, IUnicodeMapper <TLetter> mapper, TLetter?eof, RangeSet <TLetter> validRanges, out AlphabetBuilder <TLetter> builder)
where TLetter : struct, IEquatable <TLetter>, IComparable <TLetter>
{
this.output.WriteLine("Input regex (Case Sensitive: {0}, EOF letter: {1}):", caseSensitive, eof.HasValue);
this.output.WriteLine(pattern);
var provider = new UnicodeCharSetProvider();
var regex = RegexParser.Parse(pattern, null).ToInvariant(mapper, provider, caseSensitive);
this.output.WriteLine("");
this.output.WriteLine("{0} regex:", typeof(TLetter).Name);
this.output.WriteLine(regex.ToString());
builder = new AlphabetBuilder <TLetter>(regex, eof, validRanges);
this.output.WriteLine("");
this.output.WriteLine("Generated letter mapping:");
foreach (var pair in builder.AlphabetById)
{
this.output.WriteLine("{0}: {1} ({2})", pair.Key, pair.Value, pair.Value.Count);
}
this.output.WriteLine("");
this.output.WriteLine("Letter Regex:");
this.output.WriteLine(builder.Expression.ToString());
this.output.WriteLine("");
this.output.WriteLine("Mapping function pseudocode:");
var inSwitch = false;
foreach (var grouping in builder
.AlphabetById
.SelectMany(p => p.Value.Select(r => new KeyValuePair <Range <TLetter>, LetterId>(r, p.Key)))
.GroupBy(p => new {
Range = (!typeof(TLetter).IsPrimitive) || p.Key.Expand().Skip(2).Any(),
LetterId = p.Value
}, p => p.Key)
.OrderBy(p => p.Key.Range)
.ThenBy(p => p.Key.LetterId))
{
if (grouping.Key.Range)
{
if (inSwitch)
{
this.output.WriteLine("}");
inSwitch = false;
}
this.output.WriteLine("if ({0}) return {1}", string.Join(" ||" + Environment.NewLine + " ", grouping.OrderBy(r => r.From).Select(r => r.From.CompareTo(r.To) == 0 ? $"(v == '{r.From}')" : $"(v >= '{r.From}' && v <= '{r.To}')")), grouping.Key.LetterId);
}
else
{
if (!inSwitch)
{
this.output.WriteLine("switch (v) {");
inSwitch = true;
}
this.output.WriteLine("{0}" + Environment.NewLine + " return {1}", string.Join(Environment.NewLine, grouping.SelectMany(g => g.Expand()).OrderBy(r => r).Select(r => $" case '{r}':")), grouping.Key.LetterId);
}
}
if (inSwitch)
{
this.output.WriteLine("}");
}
return(builder.AlphabetById.Count);
}
public SemanticParserGrammar(IUnicodeMapper <TInput> mapper, TInput?eof = null) : this(new SemanticParserGrammarBuilder <TAstNode, TInput, TPosition>(mapper, eof))
{
}
public JsonGrammar(IUnicodeMapper <TInput> mapper, TInput?eof = null) : base(mapper, eof)
{
}
public SemanticParserGrammarBuilder(IUnicodeMapper <TInput> mapper, TInput?eof)
{
string GetGrammarKeyForDisplay()
{
return($"typeof({typeof(TAstNode).FullName})");
}
string MemberInfoForDisplay(MethodBase member)
{
return(member == null ? "(assembly)" : $"{member.DeclaringType.FullName}.{member.Name}");
}
var errors = new List <Exception>();
try {
var parts = SemanticParserGrammar <TAstNode, TInput, TPosition> .FindGrammarParts()
.OrderByDescending(p => p.Key.GetType().Name)
.ThenBy(p => (p.Key as GrammarSymbolAttribute)?.SymbolName ?? (p.Key as CharsetAttribute)?.CharsetName ?? "")
.ToList();
// Compute charsets
var charsetQueue = new Queue <KeyValuePair <string, CharsetNode> >(parts
.Select(p => p.Key)
.OfType <CharsetAttribute>()
.Select(a => new KeyValuePair <string, CharsetNode>(a.CharsetName, CharsetParser.Parse(a.CharsetExpression))));
var charsets = charsetQueue
.SelectMany(p => p.Value.GetCharsetNames())
.Except(charsetQueue.Select(p => p.Key), StringComparer.OrdinalIgnoreCase)
.ToDictionary(n => n, UnicodeRanges.FromUnicodeName, StringComparer.OrdinalIgnoreCase);
var provider = new UnicodeCharSetProvider(charsets);
var skipCount = 0;
while (charsetQueue.Count > 0)
{
var current = charsetQueue.Dequeue();
if (current.Value.GetCharsetNames().All(charsets.ContainsKey))
{
charsets.Add(current.Key, current.Value.Compute(provider));
skipCount = 0;
}
else
{
charsetQueue.Enqueue(current);
if (skipCount++ > charsetQueue.Count)
{
errors.Add(new InvalidOperationException($"The charsets cannot be computed because {String.Join(", ", charsetQueue.Select(p => p.Key))} contain circular references"));
break;
}
}
}
// Gather symbol information
var startsymbol = parts.Select(p => p.Key).OfType <StartSymbolAttribute>().SingleOrDefault();
if (startsymbol == null)
{
errors.Add(new InvalidOperationException($"Start symbol has not been defined: [assembly: StartSymbol({GetGrammarKeyForDisplay()}, ...)]"));
}
foreach (var symbol in parts
.Select(p => p.Key)
.OfType <GrammarSymbolAttribute>()
.GroupBy(a => a.SymbolName, a => a.SymbolKind, StringComparer.OrdinalIgnoreCase))
{
if (symbol.Distinct().Skip(1).Any())
{
errors.Add(new InvalidOperationException($"The symbol {symbol.Key} must not be defined as both terminal and nonterminal"));
}
else if (StringComparer.OrdinalIgnoreCase.Equals(symbol.Key, startsymbol?.SymbolName) && (symbol.First() != SymbolKind.Nonterminal))
{
errors.Add(new InvalidOperationException($"The start symbol {symbol.Key} must be a nonterminal"));
}
this.symbolsByName.Add(symbol.Key, this.symbolsByName.Count + 1);
}
SymbolId GetSymbol(string symbolName)
{
if (this.symbolsByName.TryGetValue(symbolName, out var id))
{
return(id);
}
errors.Add(new InvalidOperationException($"The symbol {symbolName} has not been defined. If the symbol name is correct, define it as virtual: [assembly: VirtualSymbol({GetGrammarKeyForDisplay()}, ...)]"));
return(SymbolId.Eof);
}
MethodBase PopulateGenericArguments(MethodBase methodBase, GrammarSymbolAttribute attribute)
{
var genericTypeParameters = attribute.GenericTypeParameters;
if (methodBase?.DeclaringType.IsGenericTypeDefinition == true)
{
var typeGenericArguments = methodBase.DeclaringType.GetGenericArguments();
if (genericTypeParameters.Length < typeGenericArguments.Length)
{
errors.Add(new InvalidOperationException($"Missing type generic arguments for {attribute} on {MemberInfoForDisplay(methodBase)}"));
return(methodBase);
}
var genericType = methodBase.DeclaringType.MakeGenericType(genericTypeParameters.Take(typeGenericArguments.Length).ToArray());
genericTypeParameters = genericTypeParameters.Skip(typeGenericArguments.Length).ToArray();
IReadOnlyDictionary <Type, Type> genericArgumentMap = genericType.GetGenericArguments().Select((t, ix) => new KeyValuePair <Type, Type>(typeGenericArguments[ix], t)).ToDictionary(p => p.Key, p => p.Value);
var mappedParameters = methodBase.GetParameters().Select(p => genericArgumentMap.GetValueOrDefault(p.ParameterType, p.ParameterType)).ToArray();
if (methodBase is ConstructorInfo)
{
methodBase = genericType.GetConstructor(mappedParameters);
}
else
{
methodBase = genericType.GetMethod(methodBase.Name, BindingFlags.Static | BindingFlags.Public, null, mappedParameters, null);
}
}
if (methodBase is MethodInfo method && method.IsGenericMethodDefinition)
{
if (method.GetGenericArguments().Length != genericTypeParameters.Length)
{
errors.Add(new InvalidOperationException($"Invalid number of method generic arguments for {attribute} on {MemberInfoForDisplay(methodBase)}"));
}
methodBase = method.MakeGenericMethod(genericTypeParameters);
}
else if (genericTypeParameters.Length > 0)
{
errors.Add(new InvalidOperationException($"Excess generic arguments for {attribute} on {MemberInfoForDisplay(methodBase)}"));
}
return(methodBase);
}
public LexerBuilder(IUnicodeMapper <TLetter> mapper, TLetter?eof = null, CharSetProviderBase provider = null)
{
this.eof = eof;
this.provider = provider ?? new UnicodeCharSetProvider();
this.mapper = mapper;
}
public Context(IUnicodeMapper <TLetter> mapper, IRangeSetProvider <Codepoint> charSetProvider, bool caseSensitive)
{
this.Mapper = mapper;
this.CharSetProvider = charSetProvider;
this.CaseSensitive = caseSensitive;
}
public RxNode <TLetter> ToInvariant <TLetter>(IUnicodeMapper <TLetter> mapper, IRangeSetProvider <Codepoint> provider, bool caseSensitive)
where TLetter : IEquatable <TLetter>, IComparable <TLetter>
{
return(this.Visit(ToInvariantVisitor <TLetter> .Default, new ToInvariantVisitor <TLetter> .Context(mapper, provider, caseSensitive)));
}
