private int Test <TLetter>(string pattern, bool caseSensitive, IUnicodeMapper <TLetter> mapper, TLetter?eof, RangeSet <TLetter> validRanges, out AlphabetBuilder <TLetter> builder) where TLetter : struct, IEquatable <TLetter>, IComparable <TLetter> { this.output.WriteLine("Input regex (Case Sensitive: {0}, EOF letter: {1}):", caseSensitive, eof.HasValue); this.output.WriteLine(pattern); var provider = new UnicodeCharSetProvider(); var regex = RegexParser.Parse(pattern, null).ToInvariant(mapper, provider, caseSensitive); this.output.WriteLine(""); this.output.WriteLine("{0} regex:", typeof(TLetter).Name); this.output.WriteLine(regex.ToString()); builder = new AlphabetBuilder <TLetter>(regex, eof, validRanges); this.output.WriteLine(""); this.output.WriteLine("Generated letter mapping:"); foreach (var pair in builder.AlphabetById) { this.output.WriteLine("{0}: {1} ({2})", pair.Key, pair.Value, pair.Value.Count); } this.output.WriteLine(""); this.output.WriteLine("Letter Regex:"); this.output.WriteLine(builder.Expression.ToString()); this.output.WriteLine(""); this.output.WriteLine("Mapping function pseudocode:"); var inSwitch = false; foreach (var grouping in builder .AlphabetById .SelectMany(p => p.Value.Select(r => new KeyValuePair <Range <TLetter>, LetterId>(r, p.Key))) .GroupBy(p => new { Range = (!typeof(TLetter).IsPrimitive) || p.Key.Expand().Skip(2).Any(), LetterId = p.Value }, p => p.Key) .OrderBy(p => p.Key.Range) .ThenBy(p => p.Key.LetterId)) { if (grouping.Key.Range) { if (inSwitch) { this.output.WriteLine("}"); inSwitch = false; } this.output.WriteLine("if ({0}) return {1}", string.Join(" ||" + Environment.NewLine + " ", grouping.OrderBy(r => r.From).Select(r => r.From.CompareTo(r.To) == 0 ? $"(v == '{r.From}')" : $"(v >= '{r.From}' && v <= '{r.To}')")), grouping.Key.LetterId); } else { if (!inSwitch) { this.output.WriteLine("switch (v) {"); inSwitch = true; } this.output.WriteLine("{0}" + Environment.NewLine + " return {1}", string.Join(Environment.NewLine, grouping.SelectMany(g => g.Expand()).OrderBy(r => r).Select(r => $" case '{r}':")), grouping.Key.LetterId); } } if (inSwitch) { this.output.WriteLine("}"); } return(builder.AlphabetById.Count); }
public SemanticParserGrammar(IUnicodeMapper <TInput> mapper, TInput?eof = null) : this(new SemanticParserGrammarBuilder <TAstNode, TInput, TPosition>(mapper, eof)) { }
public JsonGrammar(IUnicodeMapper <TInput> mapper, TInput?eof = null) : base(mapper, eof) { }
public SemanticParserGrammarBuilder(IUnicodeMapper <TInput> mapper, TInput?eof) { string GetGrammarKeyForDisplay() { return($"typeof({typeof(TAstNode).FullName})"); } string MemberInfoForDisplay(MethodBase member) { return(member == null ? "(assembly)" : $"{member.DeclaringType.FullName}.{member.Name}"); } var errors = new List <Exception>(); try { var parts = SemanticParserGrammar <TAstNode, TInput, TPosition> .FindGrammarParts() .OrderByDescending(p => p.Key.GetType().Name) .ThenBy(p => (p.Key as GrammarSymbolAttribute)?.SymbolName ?? (p.Key as CharsetAttribute)?.CharsetName ?? "") .ToList(); // Compute charsets var charsetQueue = new Queue <KeyValuePair <string, CharsetNode> >(parts .Select(p => p.Key) .OfType <CharsetAttribute>() .Select(a => new KeyValuePair <string, CharsetNode>(a.CharsetName, CharsetParser.Parse(a.CharsetExpression)))); var charsets = charsetQueue .SelectMany(p => p.Value.GetCharsetNames()) .Except(charsetQueue.Select(p => p.Key), StringComparer.OrdinalIgnoreCase) .ToDictionary(n => n, UnicodeRanges.FromUnicodeName, StringComparer.OrdinalIgnoreCase); var provider = new UnicodeCharSetProvider(charsets); var skipCount = 0; while (charsetQueue.Count > 0) { var current = charsetQueue.Dequeue(); if (current.Value.GetCharsetNames().All(charsets.ContainsKey)) { charsets.Add(current.Key, current.Value.Compute(provider)); skipCount = 0; } else { charsetQueue.Enqueue(current); if (skipCount++ > charsetQueue.Count) { errors.Add(new InvalidOperationException($"The charsets cannot be computed because {String.Join(", ", charsetQueue.Select(p => p.Key))} contain circular references")); break; } } } // Gather symbol information var startsymbol = parts.Select(p => p.Key).OfType <StartSymbolAttribute>().SingleOrDefault(); if (startsymbol == null) { errors.Add(new InvalidOperationException($"Start symbol has not been defined: [assembly: StartSymbol({GetGrammarKeyForDisplay()}, ...)]")); } foreach (var symbol in parts .Select(p => p.Key) .OfType <GrammarSymbolAttribute>() .GroupBy(a => a.SymbolName, a => a.SymbolKind, StringComparer.OrdinalIgnoreCase)) { if (symbol.Distinct().Skip(1).Any()) { errors.Add(new InvalidOperationException($"The symbol {symbol.Key} must not be defined as both terminal and nonterminal")); } else if (StringComparer.OrdinalIgnoreCase.Equals(symbol.Key, startsymbol?.SymbolName) && (symbol.First() != SymbolKind.Nonterminal)) { errors.Add(new InvalidOperationException($"The start symbol {symbol.Key} must be a nonterminal")); } this.symbolsByName.Add(symbol.Key, this.symbolsByName.Count + 1); } SymbolId GetSymbol(string symbolName) { if (this.symbolsByName.TryGetValue(symbolName, out var id)) { return(id); } errors.Add(new InvalidOperationException($"The symbol {symbolName} has not been defined. If the symbol name is correct, define it as virtual: [assembly: VirtualSymbol({GetGrammarKeyForDisplay()}, ...)]")); return(SymbolId.Eof); } MethodBase PopulateGenericArguments(MethodBase methodBase, GrammarSymbolAttribute attribute) { var genericTypeParameters = attribute.GenericTypeParameters; if (methodBase?.DeclaringType.IsGenericTypeDefinition == true) { var typeGenericArguments = methodBase.DeclaringType.GetGenericArguments(); if (genericTypeParameters.Length < typeGenericArguments.Length) { errors.Add(new InvalidOperationException($"Missing type generic arguments for {attribute} on {MemberInfoForDisplay(methodBase)}")); return(methodBase); } var genericType = methodBase.DeclaringType.MakeGenericType(genericTypeParameters.Take(typeGenericArguments.Length).ToArray()); genericTypeParameters = genericTypeParameters.Skip(typeGenericArguments.Length).ToArray(); IReadOnlyDictionary <Type, Type> genericArgumentMap = genericType.GetGenericArguments().Select((t, ix) => new KeyValuePair <Type, Type>(typeGenericArguments[ix], t)).ToDictionary(p => p.Key, p => p.Value); var mappedParameters = methodBase.GetParameters().Select(p => genericArgumentMap.GetValueOrDefault(p.ParameterType, p.ParameterType)).ToArray(); if (methodBase is ConstructorInfo) { methodBase = genericType.GetConstructor(mappedParameters); } else { methodBase = genericType.GetMethod(methodBase.Name, BindingFlags.Static | BindingFlags.Public, null, mappedParameters, null); } } if (methodBase is MethodInfo method && method.IsGenericMethodDefinition) { if (method.GetGenericArguments().Length != genericTypeParameters.Length) { errors.Add(new InvalidOperationException($"Invalid number of method generic arguments for {attribute} on {MemberInfoForDisplay(methodBase)}")); } methodBase = method.MakeGenericMethod(genericTypeParameters); } else if (genericTypeParameters.Length > 0) { errors.Add(new InvalidOperationException($"Excess generic arguments for {attribute} on {MemberInfoForDisplay(methodBase)}")); } return(methodBase); }
public LexerBuilder(IUnicodeMapper <TLetter> mapper, TLetter?eof = null, CharSetProviderBase provider = null) { this.eof = eof; this.provider = provider ?? new UnicodeCharSetProvider(); this.mapper = mapper; }
public Context(IUnicodeMapper <TLetter> mapper, IRangeSetProvider <Codepoint> charSetProvider, bool caseSensitive) { this.Mapper = mapper; this.CharSetProvider = charSetProvider; this.CaseSensitive = caseSensitive; }
public RxNode <TLetter> ToInvariant <TLetter>(IUnicodeMapper <TLetter> mapper, IRangeSetProvider <Codepoint> provider, bool caseSensitive) where TLetter : IEquatable <TLetter>, IComparable <TLetter> { return(this.Visit(ToInvariantVisitor <TLetter> .Default, new ToInvariantVisitor <TLetter> .Context(mapper, provider, caseSensitive))); }